diff --git a/.env.example b/.env.example index dbcbaaf..3c61113 100644 --- a/.env.example +++ b/.env.example @@ -34,3 +34,11 @@ AI_REQUEST_TIMEOUT=30s AI_SEARCH_TIMEOUT=65s AI_MAX_CONCURRENT=4 AI_REQUESTS_PER_MINUTE=10 + +# Fork 依赖 Git 和 git-lfs;普通文件在线读取限制不影响 Git 快照复制。 +FORK_GIT_EXECUTABLE=git +FORK_TIMEOUT=5m +FORK_MAX_CONCURRENT=2 +FORK_MAX_TEMPORARY_BYTES=5368709120 +# 可选:独立临时磁盘目录,默认系统临时目录下 community-fork。 +# FORK_TEMPORARY_DIRECTORY=/var/tmp/community-fork diff --git a/README.md b/README.md index 39f3e5e..9386df7 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ bash scripts/run-local.sh ## 环境与日志 -配置与日志方式参考 cadworld:application.yml 保存公共配置,application-test.yml 和 application-prod.yml 分别保存环境相关的数据库、Gitea 和日志配置。不指定环境时默认 test,通过 SPRING_PROFILES_ACTIVE 或启动参数切换。 +配置与日志方式参考 cadworld:application.yml 保存公共配置,application-test.yml 和 application-prod.yml 分别保存环境相关的数据库、Gitea、跨域白名单和日志配置。不指定环境时默认 test,通过 SPRING_PROFILES_ACTIVE 或启动参数切换。 prod 启动时检查数据库连接、用户名、密码、Gitea 地址和 Token 是否已配置且非空;缺失时停止启动,错误信息只包含配置项名称。 @@ -53,6 +53,7 @@ prod 启动时检查数据库连接、用户名、密码、Gitea 地址和 Token | --- | --- | --- | | 数据库 | 默认本机 community,可用 DB_* 覆盖 | 必须配置 DB_URL、DB_USERNAME、DB_PASSWORD | | Gitea | 默认本机地址,使用 GITEA_* 覆盖 | 必须配置 GITEA_BASE_URL、GITEA_TOKEN;组织默认 community | +| API 跨域来源 | `https://dev-vibemake.robotquan.com` | `https://vibemake.robotquan.com` | | OSS 封面目录 | vibemake_dev/project-covers/ | vibemake/project-covers/ | | OSS URDF 目录 | vibemake_dev/urdf/{projectCode}/{batchCode}/ | vibemake/urdf/{projectCode}/{batchCode}/ | | 连接池 | 最少 1、最多 10 个连接 | 最少 1、最多 20 个连接 | @@ -67,6 +68,8 @@ SPRING_PROFILES_ACTIVE=prod java -Djava.awt.headless=true -jar community-api/tar 本机脚本支持传递启动参数,例如 `bash scripts/run-local.sh --spring.profiles.active=prod`,此时加载 .env 中的连接信息并使用 prod 配置。测试和生产数据库、Gitea 组织需要各自隔离时,分别设置对应的 DB_*、GITEA_* 变量。 +跨域白名单由 `community.cors.allowed-origins` 指定,仅作用于 `/api/**`。支持 Bearer Token、JSON、文件上传下载和 AI SSE,预检缓存 3600 秒,暴露 `X-Request-Id`、`Content-Disposition`,不启用跨域 Cookie 凭据。启动日志会记录生效的白名单。默认不开放 HTTP、localhost、其他端口或跨环境来源;本地继续使用前端开发代理,配置见[开发代理示例](docs/community-api.md#开发代理示例)。部署代理需透传预检请求、Origin 和后端 CORS 响应头,避免重复添加跨域头;OSS Bucket 的 CORS 仍需独立配置。 + 日志配置沿用 cadworld-common 的 logback-spring.xml 模板,放在 community-common;应用名为 community-api。测试和生产默认都写入 logs,可用 LOG_PATH 覆盖目录;目录相对于应用启动时的工作目录。本机启动脚本从项目根目录运行。 同时输出控制台、community-api.log 和 community-api-error.log,日志时间使用 Asia/Shanghai,格式包含线程、级别、requestId 和类名。应用日志按天或每 100MB 归档,保留 30 天、总归档上限 10GB;错误日志仅接收 ERROR,按天或每 50MB 归档,保留 60 天、总归档上限 5GB。归档写入 archive 子目录并压缩为 gzip。SQL Mapper 和 Spring Web 保持 INFO,不记录凭据、邮箱或代码内容。 @@ -113,7 +116,7 @@ Bucket 需要配置前端实际域名的 CORS,允许 POST;图片通过公开 查找时仅检索已发布且仓库就绪项目的名称、描述、类型:先理解需求,再查询最多20个候选,最后排序并校验最多5个项目卡片。项目编码、引用原文和当前发布状态由后端校验;缺少依据的明确要求不算匹配,未证实的偏好单独列出。 -每次模型调用默认30秒,整个处理65秒(含仓库操作);`AI_REQUEST_TIMEOUT`、`AI_SEARCH_TIMEOUT` 可调整。`AI_MAX_CONCURRENT` 默认4,`AI_REQUESTS_PER_MINUTE` 默认每用户10次,均为单实例内存控制,超限不排队。连接断开时尽力取消上游调用,实际工作结束后释放并发名额。写入开始后取消或超时不保证回滚;错误事件保留初始化异常中的 projectCode,先核对工作区,不自动重发创建或 Fork。AI 错误不影响普通项目搜索。前端请求和 SSE 解析示例见接口文档的“AI 项目助手”。 +每次模型调用默认30秒,模型/搜索阶段65秒;进入 Fork 阶段后采用独立5分钟预算并保留15秒入口及终态余量,前端和代理读取超时至少400秒;`AI_REQUEST_TIMEOUT`、`AI_SEARCH_TIMEOUT` 可调整。`AI_MAX_CONCURRENT` 默认4,`AI_REQUESTS_PER_MINUTE` 默认每用户10次,均为单实例内存控制,超限不排队。连接断开时尽力取消上游调用,实际工作结束后释放并发名额。写入开始后取消或超时不保证回滚;错误事件保留初始化异常中的 projectCode,先核对工作区,不自动重发创建或 Fork。AI 错误不影响普通项目搜索。前端请求和 SSE 解析示例见接口文档的“AI 项目助手”。 本期不读取 README、代码或评论,不建立向量索引,不修改四表结构。检索效果依赖项目描述,建议写清用途、技术栈、主要硬件和适用场景。描述修改即时生效;工作区文件提交仍须重新发布才更新社区代码。 @@ -125,6 +128,26 @@ python3 scripts/test-ai.py 脚本在内存 H2 中建立公开示例项目,验证四足检索及多轮改为轮式,默认 Maven 测试跳过该联调。密钥从当前项目 `.env` 或环境变量读取,脚本不输出密钥、不写真实数据库、不访问 Gitea。 +## Git 快照 Fork + +后端保留鉴权、项目编码、来源记录和 OSS 复制,仓库数据通过 Git 对象级传输。按发布 SHA 的根 tree 生成一个无父提交,只推送新 main;保留普通大文件、文件模式、符号链接和子模块引用,不继承上游历史、其他分支或标签。子模块依赖仓库不递归复制。LFS 仅传输发布快照所需实体,推送后从新副本独立下载校验;不会只有指针却缺少实际对象。 + +应用运行用户的 PATH 需要包含 Git 和 git-lfs(本机验证 Git 2.52 / git-lfs 3.8)。CI 同样安装这两个程序,执行本地 Git 仓库及模拟 LFS 协议测试;普通业务测试替换仓库传输边界,不访问外部服务。启动时检查依赖,缺失仅使 Fork 返回 FORK_UNAVAILABLE,不阻止普通创建或其他接口。 + +开发环境可通过 `brew install git git-lfs` 安装;Debian/Ubuntu 的 CI 或部署镜像安装 `git`、`git-lfs` 软件包。以应用运行用户执行 `git --version` 和 `git lfs version` 确认 PATH 生效,无需执行全局 `git lfs install`。LFS 传输通过任务内的本地端点校验转发,拒绝 HTTP 重定向及指向其他地址的 action URL;临时目录禁止写入 git-lfs 原始错误报告,防止环境凭据落盘。 + +| 配置 | 默认值 | 用途 | +| --- | --- | --- | +| FORK_GIT_EXECUTABLE | git | Git 可执行文件路径,git-lfs 从子进程 PATH 查找 | +| FORK_TIMEOUT | 5m | Git、LFS、核验和 OSS 共用的总时间预算 | +| FORK_MAX_CONCURRENT | 2 | 单实例并发上限,实际工作结束后释放名额 | +| FORK_MAX_TEMPORARY_BYTES | 5368709120 | 每次临时磁盘上限,含 LFS 源缓存与副本校验缓存 | +| FORK_TEMPORARY_DIRECTORY | 系统临时目录/community-fork | 建议使用容量充足的独立临时磁盘 | + +服务账号的 GITEA_TOKEN 需同时允许 REST 仓库创建、HTTP Git 读写及 LFS 读写。凭据通过进程环境注入,只用于配置的 Gitea 地址;不写入 URL、命令参数、Git 配置或日志。Fork 接口继续同步返回,前端和反向代理读取超时至少400秒;AI 进入 forking 后切换专用预算并继续发送心跳。 + +Fork 不受普通在线文件读取的10 MiB限制,文件浏览、下载和 LFS 在线展开能力仍维持原规则。失败时先核对响应 data.projectCode:明确失败为 FAILED,远端写入或数据库状态待核对为 CREATING;不会自动重发、删除远端仓库或 OSS 对象。完成后清理本地临时目录。使用 scripts/test-gitea.py 可执行显式真实 Gitea 联调,仅创建并清理本次测试专用仓库,不修改已有社区项目。 + ## 运行约定 项目名称、描述、类型、封面修改直接更新展示;工作区代码提交后,社区仍读取已发布 SHA,需要手动更新。Fork 复制已发布文件快照,不继承上游提交历史,也不自动同步。项目删除采用逻辑删除,保留仓库和已有 Fork。 @@ -162,4 +185,4 @@ bash scripts/import-github-projects.sh --excel "$IMPORT_EXCEL" --execute --owner 报告逐项包含 Sheet、行号、短编码、阶段、错误码和兼容性提醒。退出码 0 表示本次选定工作完成,1 表示命令、配置或持久化等致命错误,2 表示存在预检未通过、失败或待核对项目。在线仓库或图片失效会减少实际导入量;报告中的候选数仍与静态筛选对账。 -本次只完成仓库迁移、关联和发布。社区仍有单文件读取限制,LFS 实体读取、子模块及符号链接的浏览/Fork 能力没有扩展;当前发布快照检测到的限制会写入报告。子模块只保留引用,不递归迁移独立仓库,也不下载网盘或其他外链资源。保留所有分支不代表社区提供分支切换界面。 +本次只完成仓库迁移、关联和发布。在线浏览和下载仍有单文件限制,LFS 实体、子模块及符号链接的在线浏览能力未扩展;Git 快照 Fork 已支持大文件、LFS 实体、符号链接及子模块引用,依赖仓库仍不递归复制。当前发布快照检测到的限制会写入报告。子模块只保留引用,不递归迁移独立仓库,也不下载网盘或其他外链资源。保留所有分支不代表社区提供分支切换界面。 diff --git a/community-api/src/main/resources/application-prod.yml b/community-api/src/main/resources/application-prod.yml index f42e117..b402050 100644 --- a/community-api/src/main/resources/application-prod.yml +++ b/community-api/src/main/resources/application-prod.yml @@ -11,6 +11,9 @@ spring: connection-timeout: 5000 community: + cors: + allowed-origins: + - https://vibemake.robotquan.com oss: root-directory: vibemake gitea: diff --git a/community-api/src/main/resources/application-test.yml b/community-api/src/main/resources/application-test.yml index 935d2b2..a47ea18 100644 --- a/community-api/src/main/resources/application-test.yml +++ b/community-api/src/main/resources/application-test.yml @@ -11,6 +11,9 @@ spring: connection-timeout: 5000 community: + cors: + allowed-origins: + - https://dev-vibemake.robotquan.com oss: root-directory: vibemake_dev gitea: diff --git a/community-api/src/main/resources/application.yml b/community-api/src/main/resources/application.yml index 8f2a122..ec01f12 100644 --- a/community-api/src/main/resources/application.yml +++ b/community-api/src/main/resources/application.yml @@ -55,6 +55,12 @@ community: gitea: connect-timeout: 5s request-timeout: 30s + fork: + git-executable: ${FORK_GIT_EXECUTABLE:git} + timeout: ${FORK_TIMEOUT:5m} + max-concurrent: ${FORK_MAX_CONCURRENT:2} + max-temporary-bytes: ${FORK_MAX_TEMPORARY_BYTES:5368709120} + temporary-directory: ${FORK_TEMPORARY_DIRECTORY:${java.io.tmpdir}/community-fork} springdoc: swagger-ui: path: /swagger-ui.html diff --git a/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java b/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java index 5840f41..1752f81 100644 --- a/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java +++ b/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java @@ -21,6 +21,7 @@ import com.linkerbot.community.service.ProjectService; import com.linkerbot.community.service.ai.AiModelClient; import com.linkerbot.community.service.ai.AiProjectAssistant; import com.linkerbot.community.service.config.AiProperties; +import com.linkerbot.community.service.config.ForkProperties; import com.linkerbot.community.service.gitea.GiteaClient.Change; import com.linkerbot.community.service.mapper.ProjectMapper; import com.linkerbot.community.service.mapper.UserMapper; @@ -61,6 +62,8 @@ import org.springframework.test.web.servlet.*; class AiSearchTest { @TestConfiguration static class Config { + @Bean @Primary FakeGitSnapshotCopier fakeSnapshots(FakeGiteaClient gitea) { return new FakeGitSnapshotCopier(gitea); } + @Bean @Primary FakeModel fakeModel() { return new FakeModel(); } @Bean @Primary FakeGiteaClient fakeGitea() { return new FakeGiteaClient(); } } @@ -86,12 +89,17 @@ class AiSearchTest { @Autowired FakeGiteaClient gitea; @Autowired ProjectService projectService; @Autowired AiProjectAssistant assistant; + @Autowired FakeGitSnapshotCopier snapshots; + @Autowired ForkProperties forkProperties; private User user; private String token; @BeforeEach void reset() throws Exception { SchemaSupport.reset(jdbc); gitea.reset(); + snapshots.available = true; + snapshots.beforePush = null; + forkProperties.setTimeout(Duration.ofMinutes(5)); user = new User(); user.setId(USER_SEQUENCE.incrementAndGet()); user.setUserCode(Codes.next()); @@ -178,6 +186,28 @@ class AiSearchTest { return events.getLast(); } + @Test + void crossOriginSseKeepsCorsHeadersAcrossAsyncDispatch() throws Exception { + String origin = "https://dev-vibemake.robotquan.com"; + MvcResult initial = mvc.perform(post("/api/community/ai/chat") + .header("Origin", origin) + .header("Authorization", "Bearer " + token) + .accept("text/event-stream") + .contentType("application/json") + .content(json.writeValueAsBytes(Map.of("message", "推荐机器人项目")))) + .andExpect(status().isOk()) + .andExpect(request().asyncStarted()) + .andExpect(header().string("Access-Control-Allow-Origin", origin)) + .andReturn(); + initial.getAsyncResult(6000); + mvc.perform(asyncDispatch(initial)) + .andExpect(status().isOk()) + .andExpect(content().contentTypeCompatibleWith("text/event-stream")) + .andExpect(header().string("Access-Control-Allow-Origin", origin)) + .andExpect(header().exists("Access-Control-Expose-Headers")) + .andExpect(header().exists("X-Request-Id")); + } + private ObjectNode createIntent(String type, String name) { return intent(type, List.of()).put("action", "CREATE").put("projectName", name) .put("description", "计划用于机器人教学实验"); @@ -293,6 +323,45 @@ class AiSearchTest { .containsExactly("understanding", "forking"); } + @Test void forkGetsItsOwnDeadlineAfterUnderstandingRatherThanTheSearchTimeout() throws Exception { + Project source = publishedRepository("教学项目"); + properties.setSearchTimeout(Duration.ofMillis(150)); + forkProperties.setTimeout(Duration.ofSeconds(2)); + model.handler = (name, input) -> forkIntent(source.getProjectCode(), null); + snapshots.beforePush = () -> { + try { Thread.sleep(350); } + catch (InterruptedException error) { Thread.currentThread().interrupt(); throw new BusinessException(499, "FORK_CANCELLED", "已取消"); } + }; + JsonNode result = finish(start(Map.of("message", "Fork " + source.getProjectCode()))).getLast(); + assertThat(result.path("code").asText()).isEqualTo("OK"); + assertThat(result.path("data").path("status").asText()).isEqualTo("FORKED"); + } + + @Test void disconnectDuringForkInterruptsItsWorkerAndFinishesTheProjectState() throws Exception { + Project source = publishedRepository("取消 Fork 测试"); + model.handler = (name, input) -> forkIntent(source.getProjectCode(), null); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + snapshots.beforePush = () -> { + entered.countDown(); + try { new CountDownLatch(1).await(); } + catch (InterruptedException error) { interrupted.countDown(); Thread.currentThread().interrupt(); } + throw new BusinessException(499, "FORK_CANCELLED", "已取消"); + }; + MvcResult active = start(Map.of("message", "Fork " + source.getProjectCode())); + assertThat(entered.await(2, TimeUnit.SECONDS)).isTrue(); + MockAsyncContext context = (MockAsyncContext) active.getRequest().getAsyncContext(); + for (AsyncListener listener : context.getListeners()) listener.onError(new AsyncEvent(context, new IOException("closed"))); + assertThat(interrupted.await(2, TimeUnit.SECONDS)).isTrue(); + await().atMost(Duration.ofSeconds(2)).untilAsserted(() -> { + List copies = projects.selectList(null).stream().filter(project -> !project.getProjectCode().equals(source.getProjectCode())).toList(); + assertThat(copies).singleElement().satisfies(copy -> assertThat(copy.getRepoStatus()).isEqualTo("FAILED")); + }); + snapshots.beforePush = null; + assertThat(finish(start(Map.of("message", "Fork " + source.getProjectCode()))).getLast() + .path("data").path("status").asText()).isEqualTo("FORKED"); + } + @Test void forksCurrentPublicationRatherThanTheVersionShownPreviously() throws Exception { Project source = publishedRepository("教学项目"); String latest = gitea.commit(projectService.repository(source), diff --git a/community-api/src/test/java/com/linkerbot/community/api/CorsProfileTest.java b/community-api/src/test/java/com/linkerbot/community/api/CorsProfileTest.java new file mode 100644 index 0000000..28580bb --- /dev/null +++ b/community-api/src/test/java/com/linkerbot/community/api/CorsProfileTest.java @@ -0,0 +1,49 @@ +package com.linkerbot.community.api; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.linkerbot.community.security.CorsProperties; +import com.linkerbot.community.security.SecurityConfiguration; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.springframework.boot.context.properties.bind.Bindable; +import org.springframework.boot.context.properties.bind.Binder; +import org.springframework.boot.env.YamlPropertySourceLoader; +import org.springframework.core.env.PropertySource; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.core.io.ClassPathResource; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.web.cors.CorsConfiguration; +import org.springframework.web.cors.UrlBasedCorsConfigurationSource; + +class CorsProfileTest { + @ParameterizedTest + @CsvSource({ + "test, https://dev-vibemake.robotquan.com, https://vibemake.robotquan.com", + "prod, https://vibemake.robotquan.com, https://dev-vibemake.robotquan.com" + }) + void profilesAllowOnlyTheirOwnHttpsFrontend(String profile, String allowedOrigin, String otherOrigin) + throws Exception { + StandardEnvironment environment = new StandardEnvironment(); + YamlPropertySourceLoader loader = new YamlPropertySourceLoader(); + for (PropertySource source : loader.load(profile, + new ClassPathResource("application-" + profile + ".yml"))) { + environment.getPropertySources().addFirst(source); + } + CorsProperties properties = Binder.get(environment) + .bind("community.cors", Bindable.of(CorsProperties.class)).get(); + assertThat(properties.getAllowedOrigins()).containsExactly(allowedOrigin); + + UrlBasedCorsConfigurationSource source = + new SecurityConfiguration().corsConfigurationSource(properties); + CorsConfiguration configuration = source.getCorsConfiguration( + new MockHttpServletRequest("OPTIONS", "/api/workspace/projects")); + assertThat(configuration).isNotNull(); + assertThat(configuration.checkOrigin(allowedOrigin)).isEqualTo(allowedOrigin); + assertThat(configuration.checkOrigin(otherOrigin)).isNull(); + assertThat(configuration.checkOrigin(allowedOrigin.replace("https://", "http://"))).isNull(); + assertThat(configuration.checkOrigin(allowedOrigin + ":8443")).isNull(); + assertThat(configuration.checkOrigin("http://localhost:5173")).isNull(); + } +} diff --git a/community-api/src/test/java/com/linkerbot/community/api/CorsTest.java b/community-api/src/test/java/com/linkerbot/community/api/CorsTest.java new file mode 100644 index 0000000..3944dfd --- /dev/null +++ b/community-api/src/test/java/com/linkerbot/community/api/CorsTest.java @@ -0,0 +1,165 @@ +package com.linkerbot.community.api; + +import static org.hamcrest.Matchers.containsString; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.autoconfigure.web.servlet.MockMvcPrint; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.web.servlet.MockMvc; + +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:cors;MODE=MySQL;DB_CLOSE_DELAY=-1", + "spring.datasource.username=sa", + "spring.datasource.password=", + "spring.datasource.driver-class-name=org.h2.Driver", + "community.ai.enabled=false", + "community.oss.enabled=false", + "logging.level.org.springframework=WARN" +}) +@ActiveProfiles("test") +@AutoConfigureMockMvc(print = MockMvcPrint.NONE) +class CorsTest { + private static final String FRONTEND_ORIGIN = "https://dev-vibemake.robotquan.com"; + + @Autowired MockMvc mvc; + + @ParameterizedTest + @CsvSource({ + "GET, /api/users/me", + "HEAD, /api/workspace/projects", + "POST, /api/auth/login", + "POST, /api/community/ai/chat", + "PUT, /api/workspace/projects/example/publication", + "PATCH, /api/workspace/projects/example", + "DELETE, /api/workspace/projects/example", + "OPTIONS, /api/workspace/projects" + }) + void preflightAllowsSupportedMethodsAndHeadersWithoutAuthentication(String method, String path) + throws Exception { + mvc.perform(options(path) + .header(HttpHeaders.ORIGIN, FRONTEND_ORIGIN) + .header(HttpHeaders.ACCESS_CONTROL_REQUEST_METHOD, method) + .header(HttpHeaders.ACCESS_CONTROL_REQUEST_HEADERS, + "authorization, content-type, accept")) + .andExpect(status().isOk()) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN, FRONTEND_ORIGIN)) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_ALLOW_METHODS, containsString(method))) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_ALLOW_HEADERS, containsString("authorization"))) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_ALLOW_HEADERS, containsString("content-type"))) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_ALLOW_HEADERS, containsString("accept"))) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_MAX_AGE, "3600")) + .andExpect(header().doesNotExist(HttpHeaders.ACCESS_CONTROL_ALLOW_CREDENTIALS)); + } + + @Test + void preflightRunsBeforeBearerTokenValidation() throws Exception { + mvc.perform(options("/api/users/me") + .header(HttpHeaders.ORIGIN, FRONTEND_ORIGIN) + .header(HttpHeaders.ACCESS_CONTROL_REQUEST_METHOD, "GET") + .header(HttpHeaders.AUTHORIZATION, "Bearer invalid-test-token")) + .andExpect(status().isOk()) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN, FRONTEND_ORIGIN)); + } + + @Test + void publicResponsesExposeRequestIdAndDownloadHeaders() throws Exception { + mvc.perform(get("/api/project-types").header(HttpHeaders.ORIGIN, FRONTEND_ORIGIN)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value("OK")) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN, FRONTEND_ORIGIN)) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_EXPOSE_HEADERS, containsString("X-Request-Id"))) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_EXPOSE_HEADERS, containsString("Content-Disposition"))) + .andExpect(header().exists("X-Request-Id")) + .andExpect(header().doesNotExist(HttpHeaders.ACCESS_CONTROL_ALLOW_CREDENTIALS)); + } + + @Test + void unauthenticatedAndInvalidTokenResponsesKeepCorsHeaders() throws Exception { + mvc.perform(get("/api/users/me").header(HttpHeaders.ORIGIN, FRONTEND_ORIGIN)) + .andExpect(status().isUnauthorized()) + .andExpect(jsonPath("$.code").value("UNAUTHORIZED")) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN, FRONTEND_ORIGIN)); + + mvc.perform(get("/api/project-types") + .header(HttpHeaders.ORIGIN, FRONTEND_ORIGIN) + .header(HttpHeaders.AUTHORIZATION, "Bearer invalid-test-token")) + .andExpect(status().isUnauthorized()) + .andExpect(jsonPath("$.code").value("UNAUTHORIZED")) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN, FRONTEND_ORIGIN)); + } + + @Test + void validationErrorsKeepCorsHeaders() throws Exception { + mvc.perform(post("/api/auth/login") + .header(HttpHeaders.ORIGIN, FRONTEND_ORIGIN) + .contentType(MediaType.APPLICATION_JSON) + .content("{}")) + .andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.code").value("BAD_REQUEST")) + .andExpect(header().string(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN, FRONTEND_ORIGIN)); + } + + @ParameterizedTest + @ValueSource(strings = { + "https://untrusted.example", + "http://dev-vibemake.robotquan.com", + "https://vibemake.robotquan.com", + "http://localhost:5173", + "https://dev-vibemake.robotquan.com:8443", + "https://dev-vibemake.robotquan.com.untrusted.example", + "null" + }) + void unlistedOriginsAreRejectedForPreflightAndActualRequests(String origin) throws Exception { + mvc.perform(options("/api/workspace/projects") + .header(HttpHeaders.ORIGIN, origin) + .header(HttpHeaders.ACCESS_CONTROL_REQUEST_METHOD, "POST")) + .andExpect(status().isForbidden()) + .andExpect(header().doesNotExist(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN)); + + mvc.perform(get("/api/project-types").header(HttpHeaders.ORIGIN, origin)) + .andExpect(status().isForbidden()) + .andExpect(header().doesNotExist(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN)); + } + + @ParameterizedTest + @CsvSource({"TRACE, Authorization", "POST, X-Unapproved-Header"}) + void unsupportedMethodsOrHeadersAreRejected(String method, String requestedHeader) + throws Exception { + mvc.perform(options("/api/workspace/projects") + .header(HttpHeaders.ORIGIN, FRONTEND_ORIGIN) + .header(HttpHeaders.ACCESS_CONTROL_REQUEST_METHOD, method) + .header(HttpHeaders.ACCESS_CONTROL_REQUEST_HEADERS, requestedHeader)) + .andExpect(status().isForbidden()) + .andExpect(header().doesNotExist(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN)); + } + + @Test + void requestsWithoutOriginKeepExistingAuthenticationBehavior() throws Exception { + mvc.perform(get("/api/project-types")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value("OK")) + .andExpect(header().doesNotExist(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN)); + + mvc.perform(get("/api/users/me")) + .andExpect(status().isUnauthorized()) + .andExpect(jsonPath("$.code").value("UNAUTHORIZED")) + .andExpect(header().doesNotExist(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN)); + } + + @Test + void corsDoesNotExposeNonApiRoutes() throws Exception { + mvc.perform(get("/v3/api-docs").header(HttpHeaders.ORIGIN, FRONTEND_ORIGIN)) + .andExpect(status().isOk()) + .andExpect(header().doesNotExist(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN)); + } +} diff --git a/community-api/src/test/java/com/linkerbot/community/api/FakeGitSnapshotCopier.java b/community-api/src/test/java/com/linkerbot/community/api/FakeGitSnapshotCopier.java new file mode 100644 index 0000000..14f0263 --- /dev/null +++ b/community-api/src/test/java/com/linkerbot/community/api/FakeGitSnapshotCopier.java @@ -0,0 +1,39 @@ +package com.linkerbot.community.api; + +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.fork.ForkDeadline; +import com.linkerbot.community.service.fork.GitSnapshotCopier; +import com.linkerbot.community.service.gitea.GiteaClient; +import java.util.List; +import java.util.Map; + +/** 业务测试只替换仓库传输,真实 Git/LFS 由独立协议测试覆盖。 */ +final class FakeGitSnapshotCopier implements GitSnapshotCopier { + private final FakeGiteaClient gitea; + volatile boolean available = true; + volatile Runnable beforePush; + + FakeGitSnapshotCopier(FakeGiteaClient gitea) { this.gitea = gitea; } + + @Override public void checkAvailable() { + if (!available) throw new BusinessException(503, "FORK_UNAVAILABLE", "模拟 Git 依赖不可用"); + } + + @Override + public PreparedSnapshot prepare(GiteaClient.Repository source, String sha, String authorName, String authorCode, ForkDeadline deadline) { + deadline.check(); + Map files = gitea.snapshot(source, sha); + return new PreparedSnapshot() { + @Override public String push(GiteaClient.Repository target) { + if (beforePush != null) beforePush.run(); + deadline.beginWrite(); + List changes = files.entrySet().stream() + .map(file -> new GiteaClient.Change("create", file.getKey(), null, file.getValue())).toList(); + String commit = gitea.commit(target, changes, "Fork community snapshot", authorName, authorCode).commitSha(); + deadline.confirmWrite(); + return commit; + } + @Override public void close() {} + }; + } +} diff --git a/community-api/src/test/java/com/linkerbot/community/api/FakeGiteaClient.java b/community-api/src/test/java/com/linkerbot/community/api/FakeGiteaClient.java index acb4dd0..d16bbb6 100644 --- a/community-api/src/test/java/com/linkerbot/community/api/FakeGiteaClient.java +++ b/community-api/src/test/java/com/linkerbot/community/api/FakeGiteaClient.java @@ -45,6 +45,15 @@ final class FakeGiteaClient implements GiteaClient { return repositories.get(repository.id()); } + synchronized Map snapshot(Repository repository, String sha) { + RepoData data = repo(repository); + Map snapshot = data == null ? null : data.snapshots.get(sha); + if (snapshot == null) throw new GiteaFailure(404, "NOT_FOUND", "发布版本不存在", false); + Map copy = new TreeMap<>(); + snapshot.forEach((name, bytes) -> copy.put(name, bytes.clone())); + return copy; + } + @Override public synchronized Repository createRepository(String name, boolean initialize) { if (failNextCreate) { diff --git a/community-api/src/test/java/com/linkerbot/community/api/GiteaLiveTest.java b/community-api/src/test/java/com/linkerbot/community/api/GiteaLiveTest.java index b3df147..353ca49 100644 --- a/community-api/src/test/java/com/linkerbot/community/api/GiteaLiveTest.java +++ b/community-api/src/test/java/com/linkerbot/community/api/GiteaLiveTest.java @@ -8,9 +8,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkerbot.community.security.CaptchaService; import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.fork.ForkDeadline; +import com.linkerbot.community.service.fork.GitCommandRunner; +import com.linkerbot.community.service.gitea.GiteaClient; import org.junit.jupiter.api.*; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.api.io.TempDir; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.autoconfigure.web.servlet.MockMvcPrint; @@ -26,6 +30,10 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilde import java.net.URI; import java.net.http.*; import java.time.Duration; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; import java.util.*; @SpringBootTest( @@ -34,6 +42,8 @@ import java.util.*; "spring.datasource.username=sa", "spring.datasource.password=", "spring.datasource.driver-class-name=org.h2.Driver", + "community.ai.enabled=false", + "community.oss.enabled=false", "logging.level.root=INFO", "logging.level.org.springframework=WARN" }) @@ -60,6 +70,9 @@ class GiteaLiveTest { @Autowired ObjectMapper json; @Autowired JdbcTemplate jdbc; @Autowired CommunityProperties config; + @Autowired GitCommandRunner nativeGit; + @Autowired GiteaClient remote; + @TempDir Path gitFixtures; @BeforeEach void schema() throws Exception { @@ -128,6 +141,122 @@ class GiteaLiveTest { .asText(); } + @Test + void realSnapshotForkPreservesLargeCadGitlinksSymlinksAndLfsWithoutUnpublishedHistory() throws Exception { + String owner = login("live-snapshot-owner@example.invalid"); + String other = login("live-snapshot-fork@example.invalid"); + String source = perform(body(auth(post("/api/workspace/projects"), owner), + Map.of("name", "HyperDog OpenFish 类型快照联调", "projectType", "GENERAL")), 200).path("projectCode").asText(); + String original = perform(auth(get("/api/workspace/projects/" + source + "/tree"), owner), 200).path("commitSha").asText(); + Path staging = Files.createDirectory(gitFixtures.resolve("fixture.git")); + String base = config.getGitea().getBaseUrl().replaceAll("/+$", ""); + String url = base + "/" + config.getGitea().getOwner() + "/p_" + source + ".git"; + String basic = Base64.getEncoder().encodeToString((remote.authenticatedUsername() + ":" + config.getGitea().getToken()) + .getBytes(StandardCharsets.UTF_8)); + Map settings = new LinkedHashMap<>(); + settings.put("http." + base + "/.extraHeader", "Authorization: Basic " + basic); + settings.put("lfs.url", url + "/info/lfs"); + settings.put("lfs.pushurl", url + "/info/lfs"); + settings.put("lfs.basictransfersonly", "true"); + fixtureGit(staging, settings, "init", "--bare", "--quiet"); + fixtureGit(staging, settings, "remote", "add", "origin", url); + fixtureGit(staging, settings, "fetch", "--quiet", "origin", "refs/heads/main"); + fixtureGit(staging, settings, "read-tree", original); + String ordinary = fixtureBlob(staging, settings, "published".getBytes(StandardCharsets.UTF_8)); + String cad = fixtureBlob(staging, settings, new byte[13 * 1024 * 1024]); + String link = fixtureBlob(staging, settings, "README.md".getBytes(StandardCharsets.UTF_8)); + byte[] lfsBytes = new byte[256 * 1024]; + new Random(42).nextBytes(lfsBytes); + String oid = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(lfsBytes)); + Path object = staging.resolve("lfs/objects/" + oid.substring(0, 2) + "/" + oid.substring(2, 4) + "/" + oid); + Files.createDirectories(object.getParent()); + Files.write(object, lfsBytes); + String pointer = fixtureBlob(staging, settings, ("version https://git-lfs.github.com/spec/v1\noid sha256:" + oid + + "\nsize " + lfsBytes.length + "\n").getBytes(StandardCharsets.UTF_8)); + String attributes = fixtureBlob(staging, settings, "model.bin filter=lfs diff=lfs merge=lfs -text\n".getBytes(StandardCharsets.UTF_8)); + Map files = new LinkedHashMap<>(); + files.put("README.md", "100644," + ordinary); + files.put("说明/示例.txt", "100644," + ordinary); + files.put("run.sh", "100755," + ordinary); + files.put("SOLIDWORKS_FILES/Assembly_#1.SLDASM", "100644," + cad); + files.put("readme-link", "120000," + link); + files.put("model.bin", "100644," + pointer); + files.put(".gitattributes", "100644," + attributes); + for (String path : List.of("uros/drive_base", "uros/micro-ROS-Agent", "uros/micro_ros_msgs", "uros_pkg/src/micro_ros_setup")) + files.put(path, "160000," + "a".repeat(40)); + for (Map.Entry file : files.entrySet()) { + String[] modeAndHash = file.getValue().split(","); + fixtureGit(staging, settings, "update-index", "--add", "--cacheinfo", modeAndHash[0], modeAndHash[1], file.getKey()); + } + String tree = fixtureGit(staging, settings, "write-tree"); + String publication = fixtureGit(staging, settings, "commit-tree", tree, "-p", original, "-m", "Special Git/LFS fixture"); + fixtureGit(staging, settings, "update-ref", "refs/heads/fixture", publication); + fixtureGit(staging, settings, "lfs", "push", "--all", "origin", "refs/heads/fixture"); + fixtureGit(staging, settings, "push", "--quiet", "origin", publication + ":refs/heads/main"); + perform(body(auth(put("/api/workspace/projects/" + source + "/publication"), owner), Map.of("commitSha", publication)), 200); + perform(body(auth(post("/api/workspace/projects/" + source + "/commits"), owner), Map.of( + "expectedCommitSha", publication, "message", "unpublished change", "changes", List.of(Map.of( + "action", "UPDATE_FILE", "path", "README.md", "content", "private unpublished")))), 200); + Set copies = new HashSet<>(); + for (String token : List.of(owner, other)) { + JsonNode fork = perform(body(auth(post("/api/community/projects/" + source + "/forks"), token), + Map.of("name", "快照联调副本", "publishedCommitSha", publication)), 200); + String code = fork.path("projectCode").asText(); + assertThat(copies.add(code)).isTrue(); + assertThat(fork.path("repoStatus").asText()).isEqualTo("READY"); + assertThat(fork.path("sourceCommitSha").asText()).isEqualTo(publication); + GiteaClient.Repository repository = new GiteaClient.Repository(0, config.getGitea().getOwner(), "p_" + code); + JsonNode metadata = readRepository("p_" + code, ""); + assertThat(metadata.path("empty").asBoolean()).as("Gitea metadata empty; defaultBranch=%s", metadata.path("default_branch").asText()).isFalse(); + String head = readRepository("p_" + code, "/branches/main").path("commit").path("id").asText(); + JsonNode commit = readRepository("p_" + code, "/git/commits/" + head); + assertThat(commit.path("parents").isArray()).isTrue(); + assertThat(commit.path("parents").isEmpty()).isTrue(); + String copyRemote = "copy" + copies.size(); + fixtureGit(staging, settings, "remote", "add", copyRemote, base + "/" + config.getGitea().getOwner() + "/p_" + code + ".git"); + fixtureGit(staging, settings, "fetch", "--quiet", "--no-tags", copyRemote, "refs/heads/main"); + assertThat(fixtureGit(staging, settings, "rev-parse", head + "^{tree}")).isEqualTo(tree); + assertThat(remote.history(repository, 1, 20).total()).isEqualTo(1); + JsonNode entries = readRepository("p_" + code, "/git/trees/" + head + "?recursive=true"); + assertThat(entries.path("tree").findValuesAsText("mode")).contains("100755", "120000", "160000"); + assertThat(perform(auth(get("/api/workspace/projects/" + code + "/files").param("path", "README.md"), token), 200) + .path("content").asText()).isEqualTo("published"); + mvc.perform(auth(get("/api/workspace/projects/" + code + "/files") + .param("path", "SOLIDWORKS_FILES/Assembly_#1.SLDASM").param("download", "true"), token)).andExpect(status().is(413)); + } + } + + private String fixtureGit(Path staging, Map settings, String... args) { + GitCommandRunner.Result result = nativeGit.run(staging, new ForkDeadline(Duration.ofMinutes(2)), "live-fixture", settings, + Map.of("GIT_AUTHOR_NAME", "Fixture", "GIT_AUTHOR_EMAIL", "fixture@users.community.invalid", + "GIT_COMMITTER_NAME", "Fixture", "GIT_COMMITTER_EMAIL", "fixture@users.community.invalid"), args); + assertThat(result.exitCode()).as("live Git fixture command completed").isZero(); + return result.output(); + } + + private String fixtureBlob(Path staging, Map settings, byte[] bytes) throws Exception { + Path input = Files.createTempFile(gitFixtures, "blob-", ".bin"); + Files.write(input, bytes); + return fixtureGit(staging, settings, "hash-object", "-w", input.toString()); + } + + private JsonNode readRepository(String repository, String suffix) throws Exception { + String url = config.getGitea().getBaseUrl().replaceAll("/+$", "") + "/api/v1/repos/" + + config.getGitea().getOwner() + "/" + repository + suffix; + HttpRequest request = HttpRequest.newBuilder(URI.create(url)).timeout(Duration.ofSeconds(30)) + .header("Authorization", "token " + config.getGitea().getToken()).GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() != 200) { + String diagnostic = new String(response.body(), StandardCharsets.UTF_8) + .replace(config.getGitea().getToken(), "[redacted]") + .replaceAll("(?im)(Authorization|Cookie|Set-Cookie):[^\\r\\n]*", "$1: [redacted]"); + if (diagnostic.length() > 18000) diagnostic = diagnostic.substring(0, 18000); + assertThat(response.statusCode()).withFailMessage("Fixture Gitea response: %s", diagnostic).isEqualTo(200); + } + assertThat(response.statusCode()).isEqualTo(200); + return json.readTree(response.body()); + } + @Test void realOrganizationRepositoryCommitPublishUploadAndSnapshotFork() throws Exception { String owner = login("live-owner@example.invalid"), @@ -228,6 +357,8 @@ class GiteaLiveTest { c.path("commitSha").asText())), 200); String copy = fork.path("projectCode").asText(); + readRepository("p_" + copy, "/contents/%E6%BA%90%E7%A0%81/hello%20world.txt?ref=" + + readRepository("p_" + copy, "/branches/main").path("commit").path("id").asText()); assertThat( perform( auth( diff --git a/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java b/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java index bf0c27d..d3aa595 100644 --- a/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java +++ b/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java @@ -1,6 +1,7 @@ package com.linkerbot.community.api; import static org.assertj.core.api.Assertions.*; +import static org.hamcrest.Matchers.containsString; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; @@ -10,6 +11,7 @@ import com.linkerbot.community.security.CaptchaService; import com.linkerbot.community.service.config.CommunityProperties; import com.linkerbot.community.service.config.OssProperties; import com.linkerbot.community.service.config.UrdfProperties; +import com.linkerbot.community.common.BusinessException; import org.junit.jupiter.api.*; import org.springframework.beans.factory.annotation.Autowired; @@ -42,6 +44,8 @@ import java.util.concurrent.*; class WorkflowTest { @TestConfiguration static class Config { + @Bean @Primary FakeGitSnapshotCopier fakeSnapshots(FakeGiteaClient gitea) { return new FakeGitSnapshotCopier(gitea); } + @Bean @Primary FakeOssObjectStore fakeOss() { return new FakeOssObjectStore(); } @@ -71,11 +75,14 @@ class WorkflowTest { @Autowired OssProperties oss; @Autowired FakeOssObjectStore ossObjects; @Autowired UrdfProperties urdfLimits; + @Autowired FakeGitSnapshotCopier snapshots; @BeforeEach void reset() throws Exception { SchemaSupport.reset(jdbc); gitea.reset(); + snapshots.available = true; + snapshots.beforePush = null; oss.setEnabled(false); ossObjects.reset(); urdfLimits.setMaxFileBytes(52428800); @@ -182,6 +189,54 @@ class WorkflowTest { 200); } + @Test + void crossOriginLoginDownloadsAndBusinessErrorsKeepCorsHeaders() throws Exception { + String origin = "https://dev-vibemake.robotquan.com"; + String captchaCode = perform(get("/api/auth/captcha").header("Origin", origin), 200) + .path("data").path("captchaCode").asText(); + MvcResult loginResult = mvc.perform(body(post("/api/auth/login").header("Origin", origin), + Map.of("email", "cors-owner@example.invalid", "captchaCode", captchaCode, + "captchaAnswer", "ABCDE"))) + .andExpect(status().isOk()) + .andExpect(header().string("Access-Control-Allow-Origin", origin)) + .andExpect(jsonPath("$.code").value("OK")) + .andReturn(); + String owner = json.readTree(loginResult.getResponse().getContentAsString()) + .path("data").path("token").asText(); + String code = create(owner, "跨域下载项目"); + + mvc.perform(auth(get("/api/workspace/projects/" + code + "/files"), owner) + .header("Origin", origin) + .param("path", "README.md").param("download", "true")) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_OCTET_STREAM)) + .andExpect(header().string("Content-Disposition", containsString("attachment"))) + .andExpect(header().string("Access-Control-Allow-Origin", origin)) + .andExpect(header().string("Access-Control-Expose-Headers", containsString("Content-Disposition"))) + .andExpect(header().string("Access-Control-Expose-Headers", containsString("X-Request-Id"))); + + String other = login("cors-other@example.invalid"); + mvc.perform(auth(get("/api/workspace/projects/" + code), other).header("Origin", origin)) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.code").value("NOT_FOUND")) + .andExpect(header().string("Access-Control-Allow-Origin", origin)); + + mvc.perform(auth(get("/api/workspace/projects/missing-project"), owner).header("Origin", origin)) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.code").value("NOT_FOUND")) + .andExpect(header().string("Access-Control-Allow-Origin", origin)); + + jdbc.update("UPDATE sys_user SET status = 'DISABLED' WHERE email = ?", "cors-other@example.invalid"); + String disabledCaptchaCode = perform(get("/api/auth/captcha").header("Origin", origin), 200) + .path("data").path("captchaCode").asText(); + mvc.perform(body(post("/api/auth/login").header("Origin", origin), + Map.of("email", "cors-other@example.invalid", "captchaCode", disabledCaptchaCode, + "captchaAnswer", "ABCDE"))) + .andExpect(status().isForbidden()) + .andExpect(jsonPath("$.code").value("ACCOUNT_DISABLED")) + .andExpect(header().string("Access-Control-Allow-Origin", origin)); + } + @Test void roboticArmTypeSupportsDiscoveryCreationEditingAndCommunityFiltering() throws Exception { JsonNode types = perform(get("/api/project-types"), 200).path("data"); @@ -583,6 +638,43 @@ class WorkflowTest { "SYS_USER", "PROJECT", "PROJECT_COMMENT", "PROJECT_LIKE"); } + @Test + void unavailableForkRejectsBeforeCreatingAnyProjectRecord() throws Exception { + String token = login("fork-runtime@example.invalid"); + String source = create(token, "来源"); + String sha = head(token, source); + publish(token, source, sha); + snapshots.available = false; + JsonNode error = perform(body(auth(post("/api/community/projects/" + source + "/forks"), token), + Map.of("name", "副本", "publishedCommitSha", sha)), 503); + assertThat(error.path("code").asText()).isEqualTo("FORK_UNAVAILABLE"); + assertThat(jdbc.queryForObject("SELECT COUNT(*) FROM project", Integer.class)).isEqualTo(1); + } + + @Test + void failedAndUncertainGitCopiesPreserveProjectCodeAndRepositoryAssociation() throws Exception { + String token = login("fork-status@example.invalid"); + String source = create(token, "来源"); + String sha = head(token, source); + publish(token, source, sha); + snapshots.beforePush = () -> { throw new BusinessException(502, "FORK_COPY_FAILED", "模拟复制失败"); }; + JsonNode failed = perform(body(auth(post("/api/community/projects/" + source + "/forks"), token), + Map.of("name", "失败副本", "publishedCommitSha", sha)), 502); + String failedCode = failed.path("data").path("projectCode").asText(); + assertThat(failed.path("code").asText()).isEqualTo("FORK_COPY_FAILED"); + assertThat(jdbc.queryForObject("SELECT repo_status FROM project WHERE project_code = ?", String.class, failedCode)).isEqualTo("FAILED"); + assertThat(jdbc.queryForObject("SELECT gitea_repo_id FROM project WHERE project_code = ?", Long.class, failedCode)).isPositive(); + snapshots.beforePush = null; + gitea.timeoutNextCommit = true; + JsonNode unknown = perform(body(auth(post("/api/community/projects/" + source + "/forks"), token), + Map.of("name", "待核对副本", "publishedCommitSha", sha)), 504); + String unknownCode = unknown.path("data").path("projectCode").asText(); + assertThat(unknown.path("code").asText()).isEqualTo("REMOTE_RESULT_UNKNOWN"); + assertThat(jdbc.queryForObject("SELECT repo_status FROM project WHERE project_code = ?", String.class, unknownCode)).isEqualTo("CREATING"); + assertThat(jdbc.queryForObject("SELECT gitea_repo_id FROM project WHERE project_code = ?", Long.class, unknownCode)).isPositive(); + assertThat(jdbc.queryForObject("SELECT repo_status FROM project WHERE project_code = ?", String.class, source)).isEqualTo("READY"); + } + @Test void publishPinsFilesAndForkCopiesOnlyPublishedSnapshot() throws Exception { String owner = login("owner@example.com"); diff --git a/community-security/src/main/java/com/linkerbot/community/security/CorsProperties.java b/community-security/src/main/java/com/linkerbot/community/security/CorsProperties.java new file mode 100644 index 0000000..9d3d472 --- /dev/null +++ b/community-security/src/main/java/com/linkerbot/community/security/CorsProperties.java @@ -0,0 +1,17 @@ +package com.linkerbot.community.security; + +import lombok.Getter; +import lombok.Setter; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Getter +@Setter +@Component +@ConfigurationProperties(prefix = "community.cors") +public class CorsProperties { + private List allowedOrigins = List.of(); +} diff --git a/community-security/src/main/java/com/linkerbot/community/security/SecurityConfiguration.java b/community-security/src/main/java/com/linkerbot/community/security/SecurityConfiguration.java index cd6ea07..f25904c 100644 --- a/community-security/src/main/java/com/linkerbot/community/security/SecurityConfiguration.java +++ b/community-security/src/main/java/com/linkerbot/community/security/SecurityConfiguration.java @@ -3,8 +3,11 @@ package com.linkerbot.community.security; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkerbot.community.common.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.*; +import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.http.SessionCreationPolicy; @@ -12,9 +15,33 @@ import org.springframework.security.web.SecurityFilterChain; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; import org.springframework.security.web.context.RequestAttributeSecurityContextRepository; import org.springframework.security.web.context.SecurityContextRepository; +import org.springframework.web.cors.CorsConfiguration; +import org.springframework.web.cors.UrlBasedCorsConfigurationSource; + +import java.util.List; @Configuration public class SecurityConfiguration { + private static final Logger log = LoggerFactory.getLogger(SecurityConfiguration.class); + + @Bean + public UrlBasedCorsConfigurationSource corsConfigurationSource(CorsProperties properties) { + CorsConfiguration configuration = new CorsConfiguration(); + configuration.setAllowedOrigins(properties.getAllowedOrigins()); + configuration.setAllowedMethods( + List.of("GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS")); + configuration.setAllowedHeaders( + List.of(HttpHeaders.AUTHORIZATION, HttpHeaders.CONTENT_TYPE, HttpHeaders.ACCEPT)); + configuration.setExposedHeaders(List.of("X-Request-Id", HttpHeaders.CONTENT_DISPOSITION)); + configuration.setAllowCredentials(false); + configuration.setMaxAge(3600L); + + UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); + source.registerCorsConfiguration("/api/**", configuration); + log.info("API 跨域配置已加载 path=/api/** allowedOrigins={}", properties.getAllowedOrigins()); + return source; + } + @Bean public SecurityContextRepository requestSecurityContextRepository() { // 仅在本次请求及其异步分派内保留身份,不创建 HttpSession。 @@ -33,8 +60,11 @@ public class SecurityConfiguration { @Bean public SecurityFilterChain securityFilterChain( HttpSecurity http, BearerTokenFilter tokenFilter, ObjectMapper json, - SecurityContextRepository contextRepository) throws Exception { - http.csrf(csrf -> csrf.disable()) + SecurityContextRepository contextRepository, + UrlBasedCorsConfigurationSource corsConfigurationSource) throws Exception { + // 在 Token 鉴权前处理预检,并让鉴权失败响应同样携带跨域头。 + http.cors(cors -> cors.configurationSource(corsConfigurationSource)) + .csrf(csrf -> csrf.disable()) .sessionManagement( session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS)) .securityContext(context -> context.securityContextRepository(contextRepository)) diff --git a/community-service/src/main/java/com/linkerbot/community/service/config/ForkProperties.java b/community-service/src/main/java/com/linkerbot/community/service/config/ForkProperties.java new file mode 100644 index 0000000..aa452e9 --- /dev/null +++ b/community-service/src/main/java/com/linkerbot/community/service/config/ForkProperties.java @@ -0,0 +1,33 @@ +package com.linkerbot.community.service.config; + +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Positive; +import java.nio.file.Path; +import java.time.Duration; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; +import org.springframework.validation.annotation.Validated; + +@Getter +@Setter +@Validated +@Component +@ConfigurationProperties(prefix = "community.fork") +public class ForkProperties { + @NotBlank private String gitExecutable = "git"; + private Duration timeout = Duration.ofMinutes(5); + @Min(1) @Max(32) private int maxConcurrent = 2; + @Positive private long maxTemporaryBytes = 5L * 1024 * 1024 * 1024; + @NotBlank private String temporaryDirectory = Path.of(System.getProperty("java.io.tmpdir"), "community-fork").toString(); + + @AssertTrue(message = "Fork 超时必须在 100 毫秒至 1 小时之间") + public boolean isTimeoutValid() { + return timeout != null && timeout.compareTo(Duration.ofMillis(100)) >= 0 + && timeout.compareTo(Duration.ofHours(1)) <= 0; + } +} diff --git a/community-service/src/main/java/com/linkerbot/community/service/fork/CliGitSnapshotCopier.java b/community-service/src/main/java/com/linkerbot/community/service/fork/CliGitSnapshotCopier.java new file mode 100644 index 0000000..ee3a28b --- /dev/null +++ b/community-service/src/main/java/com/linkerbot/community/service/fork/CliGitSnapshotCopier.java @@ -0,0 +1,268 @@ +package com.linkerbot.community.service.fork; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.common.FilePaths; +import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.config.ForkProperties; +import com.linkerbot.community.service.gitea.GiteaClient; +import com.linkerbot.community.service.gitea.GiteaClient.Repository; +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.Duration; +import java.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** 使用原始 Git tree 生成根提交,不检出文件、不递归子模块、不复制上游历史。 */ +@Component +public class CliGitSnapshotCopier implements GitSnapshotCopier { + private static final Logger log = LoggerFactory.getLogger(CliGitSnapshotCopier.class); + private final ForkProperties properties; + private final CommunityProperties community; + private final GitCommandRunner commands; + private final GiteaClient gitea; + private final ObjectMapper json; + + public CliGitSnapshotCopier(ForkProperties properties, CommunityProperties community, GitCommandRunner commands, + GiteaClient gitea, ObjectMapper json) { + this.properties = properties; + this.community = community; + this.commands = commands; + this.gitea = gitea; + this.json = json; + } + + @Override + public void checkAvailable() { + try { + trustedBase(); + if (community.getGitea().getToken().isBlank()) throw unavailable(); + ForkDeadline deadline = new ForkDeadline(Duration.ofSeconds(5)); + if (commands.run(null, deadline, "git-runtime", Map.of(), Map.of(), "--version").exitCode() != 0 + || commands.run(null, deadline, "lfs-runtime", Map.of(), Map.of(), "lfs", "version").exitCode() != 0) + throw unavailable(); + Path root = Files.createDirectories(Path.of(properties.getTemporaryDirectory())); + Path probe = Files.createTempFile(root, ".fork-probe-", ".tmp"); + Files.delete(probe); + } catch (IOException | RuntimeException error) { + throw unavailable(); + } + } + + private BusinessException unavailable() { + return new BusinessException(503, "FORK_UNAVAILABLE", "Fork 不可用,请检查 Git、git-lfs、Gitea 配置和临时目录"); + } + + protected URI trustedBase() { + URI base = URI.create(community.getGitea().getBaseUrl().replaceAll("/+$", "")); + if (!("http".equals(base.getScheme()) || "https".equals(base.getScheme())) || base.getHost() == null + || base.getRawUserInfo() != null || base.getRawQuery() != null || base.getRawFragment() != null) + throw unavailable(); + return base; + } + + protected String repositoryUrl(Repository repository) { + String owner = URLEncoder.encode(repository.owner(), StandardCharsets.UTF_8).replace("+", "%20"); + String name = URLEncoder.encode(repository.name(), StandardCharsets.UTF_8).replace("+", "%20"); + return trustedBase() + "/" + owner + "/" + name + ".git"; + } + + protected Map configuration(String lfsEndpoint) { + String username = gitea.authenticatedUsername(); + if (username == null || username.isBlank()) throw unavailable(); + String basic = Base64.getEncoder().encodeToString((username + ":" + community.getGitea().getToken()).getBytes(StandardCharsets.UTF_8)); + Map settings = new LinkedHashMap<>(); + settings.put("http." + trustedBase() + "/.extraHeader", "Authorization: Basic " + basic); + settings.put("lfs.url", lfsEndpoint); + settings.put("lfs.pushurl", lfsEndpoint); + settings.put("lfs.fetchinclude", ""); + settings.put("lfs.fetchexclude", ""); + settings.put("lfs.fetchrecentalways", "false"); + settings.put("lfs.fetchrecentrefsdays", "0"); + settings.put("lfs.fetchrecentcommitsdays", "0"); + settings.put("lfs.basictransfersonly", "true"); + settings.put("lfs.standalonetransferagent", ""); + settings.put("lfs.transfer.maxretries", "1"); + settings.put("lfs.transfer.maxverifies", "1"); + settings.put("lfs.transfer.maxretrydelay", "0"); + settings.put("lfs.concurrenttransfers", "2"); + return settings; + } + + @Override + public PreparedSnapshot prepare(Repository source, String publishedSha, String authorName, String authorCode, + ForkDeadline deadline) { + String sha = FilePaths.sha(publishedSha); + Path workspace = null; + Snapshot snapshot = null; + try { + deadline.check(); + workspace = Files.createTempDirectory(Path.of(properties.getTemporaryDirectory()), "fork-" + deadline.projectCode() + "-"); + Files.setPosixFilePermissions(workspace, PosixFilePermissions.fromString("rwx------")); + String url = repositoryUrl(source); + Map settings = configuration(url + "/info/lfs"); + snapshot = new Snapshot(workspace, deadline, settings); + snapshot.require("init", "init", "--bare", "--quiet"); + blockLfsLogs(workspace.resolve("lfs")); + snapshot.require("source-remote", "remote", "add", "source", url); + GitCommandRunner.Result fetched = snapshot.run("fetch-sha", "fetch", "--quiet", "--depth=1", "--no-tags", "source", sha); + if (fetched.exitCode() != 0) { + boolean shallow = snapshot.require("shallow-check", "rev-parse", "--is-shallow-repository").equals("true"); + if (shallow) snapshot.require("fetch-main", "fetch", "--quiet", "--no-tags", "--unshallow", "source", "refs/heads/main"); + else snapshot.require("fetch-main", "fetch", "--quiet", "--no-tags", "source", "refs/heads/main"); + } + snapshot.require("source-check", "cat-file", "-e", sha + "^{commit}"); + snapshot.tree = FilePaths.sha(snapshot.require("source-tree", "rev-parse", sha + "^{tree}")); + String author = authorName == null ? "Community user" : authorName.replaceAll("[\\r\\n<>\\x00]", " ").trim(); + if (author.isEmpty()) author = "Community user"; + Map identity = Map.of("GIT_AUTHOR_NAME", author, "GIT_COMMITTER_NAME", author, + "GIT_AUTHOR_EMAIL", authorCode + "@users.community.invalid", "GIT_COMMITTER_EMAIL", authorCode + "@users.community.invalid"); + GitCommandRunner.Result commit = commands.run(workspace, deadline, "root-commit", settings, identity, + "commit-tree", snapshot.tree, "-m", "Fork community snapshot"); + if (commit.exitCode() != 0) throw GitCommandRunner.failed(deadline); + snapshot.commit = FilePaths.sha(commit.output()); + snapshot.require("snapshot-ref", "update-ref", "refs/heads/snapshot", snapshot.commit); + snapshot.require("snapshot-head", "symbolic-ref", "HEAD", "refs/heads/snapshot"); + String parents = snapshot.require("root-check", "rev-list", "--parents", "-n", "1", snapshot.commit); + if (!parents.equals(snapshot.commit)) throw GitCommandRunner.failed(deadline); + JsonNode listing = json.readTree(snapshot.require("lfs-list", "lfs", "ls-files", "--json", sha)); + JsonNode objects = listing == null ? null : listing.get("files"); + if (objects == null || !(objects.isArray() || objects.isNull())) throw GitCommandRunner.failed(deadline); + snapshot.hasLfs = objects.isArray() && !objects.isEmpty(); + if (snapshot.hasLfs) { + snapshot.protectLfs(); + snapshot.require("lfs-fetch", "lfs", "fetch", "--include=", "--exclude=", "source", sha); + snapshot.require("lfs-fsck", "lfs", "fsck", "--objects", sha); + } + log.info("Fork 快照准备完成 projectCode={} sourceRepository={} sourceCommitSha={} snapshotCommitSha={} hasLfs={}", + deadline.projectCode(), source.name(), sha, snapshot.commit, snapshot.hasLfs); + return snapshot; + } catch (IOException error) { + if (snapshot != null) snapshot.close(); + else if (workspace != null) clean(workspace, deadline.projectCode()); + throw GitCommandRunner.failed(deadline); + } catch (RuntimeException error) { + if (snapshot != null) snapshot.close(); + else if (workspace != null) clean(workspace, deadline.projectCode()); + throw error; + } + } + + private final class Snapshot implements PreparedSnapshot { + private final Path workspace; + private final ForkDeadline deadline; + private Map settings; + private String tree; + private String commit; + private boolean hasLfs; + private LfsEndpointGuard lfsGuard; + + private Snapshot(Path workspace, ForkDeadline deadline, Map settings) { + this.workspace = workspace; + this.deadline = deadline; + this.settings = settings; + } + + private GitCommandRunner.Result run(String stage, String... args) { + return commands.run(workspace, deadline, stage, settings, Map.of(), args); + } + + private String require(String stage, String... args) { + GitCommandRunner.Result result = run(stage, args); + if (result.exitCode() != 0) { + if (result.rejected()) deadline.confirmWrite(); + throw GitCommandRunner.failed(deadline); + } + return result.output(); + } + + private void protectLfs() { + if (lfsGuard != null) lfsGuard.close(); + try { + String header = settings.get("http." + trustedBase() + "/.extraHeader"); + lfsGuard = new LfsEndpointGuard(trustedBase(), URI.create(settings.get("lfs.url")), + header.substring("Authorization: ".length()), deadline, json); + settings.put("lfs.url", lfsGuard.endpoint()); + settings.put("lfs.pushurl", lfsGuard.endpoint()); + settings.put("http." + lfsGuard.endpoint() + "/.extraHeader", "Authorization: " + lfsGuard.localAuthorization()); + } catch (IOException error) { throw GitCommandRunner.failed(deadline); } + } + + @Override + public String push(Repository target) { + String url = repositoryUrl(target); + settings = configuration(url + "/info/lfs"); + if (hasLfs) protectLfs(); + require("target-remote", "remote", "add", "target", url); + if (!require("snapshot-tree", "rev-parse", commit + "^{tree}").equals(tree)) throw GitCommandRunner.failed(deadline); + if (hasLfs) { + deadline.beginWrite(); + require("lfs-push", "lfs", "push", "--all", "target", "refs/heads/snapshot"); + deadline.confirmWrite(); + } + deadline.beginWrite(); + require("git-push", "push", "--quiet", "target", commit + ":refs/heads/main"); + String remote = require("target-check", "ls-remote", "target", "refs/heads/main"); + if (!remote.equals(commit + "\trefs/heads/main")) throw GitCommandRunner.failed(deadline); + deadline.confirmWrite(); + verifyRemote(target, commit, deadline); + if (hasLfs) { + // 用独立缓存从副本下载,确认不依赖来源的本地缓存且实体完整。 + try { blockLfsLogs(workspace.resolve("verified-lfs")); } + catch (IOException error) { throw GitCommandRunner.failed(deadline); } + settings.put("lfs.storage", workspace.resolve("verified-lfs").toString()); + require("lfs-verify-fetch", "lfs", "fetch", "--include=", "--exclude=", "target", commit); + require("lfs-verify-fsck", "lfs", "fsck", "--objects", commit); + } + deadline.check(); + return commit; + } + + @Override public void close() { + if (lfsGuard != null) lfsGuard.close(); + clean(workspace, deadline.projectCode()); + } + } + + protected void verifyRemote(Repository target, String commit, ForkDeadline deadline) { + // Gitea 首次推送的元数据由服务端同步;在 empty 清除之前访问分支 API 可能失败。 + while (gitea.inspect(target.name()).empty()) { + deadline.check(); + try { Thread.sleep(Math.min(100, Math.max(1, deadline.remaining().toMillis()))); } + catch (InterruptedException error) { Thread.currentThread().interrupt(); throw deadline.failure(); } + } + if (!gitea.head(target).equals(commit)) throw GitCommandRunner.failed(deadline); + } + + private void blockLfsLogs(Path storage) throws IOException { + // git-lfs 错误报告会写入全部 GIT_* 环境,必须在任何 LFS 命令前阻止落盘。 + Files.createDirectories(storage); + Files.createFile(storage.resolve("logs")); + } + + private void clean(Path directory, String projectCode) { + try { + Files.walkFileTree(directory, new SimpleFileVisitor<>() { + @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) throws IOException { + Files.deleteIfExists(file); + return FileVisitResult.CONTINUE; + } + @Override public FileVisitResult postVisitDirectory(Path folder, IOException error) throws IOException { + if (error != null) throw error; + Files.deleteIfExists(folder); + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException error) { + log.warn("Fork 临时目录清理失败 projectCode={} errorType={}", projectCode, error.getClass().getSimpleName()); + } + } +} diff --git a/community-service/src/main/java/com/linkerbot/community/service/fork/ForkDeadline.java b/community-service/src/main/java/com/linkerbot/community/service/fork/ForkDeadline.java new file mode 100644 index 0000000..9944bdd --- /dev/null +++ b/community-service/src/main/java/com/linkerbot/community/service/fork/ForkDeadline.java @@ -0,0 +1,73 @@ +package com.linkerbot.community.service.fork; + +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.gitea.GiteaFailure; +import java.time.Duration; +import java.util.Map; +import java.util.ArrayList; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** 一次 Fork 的截止时间和远端写入状态;仅在其独立工作线程内绑定。 */ +public final class ForkDeadline { + private static final Logger log = LoggerFactory.getLogger(ForkDeadline.class); + private static final ThreadLocal CURRENT = new ThreadLocal<>(); + private final long expires; + private volatile String projectCode; + private volatile boolean writing; + private volatile String cancellation; + private final List cleanup = new ArrayList<>(); + + public ForkDeadline(Duration timeout) { expires = System.nanoTime() + timeout.toNanos(); } + + static void bind(ForkDeadline deadline) { CURRENT.set(deadline); } + static void unbind() { CURRENT.remove(); } + public static ForkDeadline current() { return CURRENT.get(); } + public void projectCode(String code) { projectCode = code; } + public String projectCode() { return projectCode; } + public boolean writing() { return writing; } + public void onClose(Runnable action) { cleanup.add(action); } + void closeResources() { + for (Runnable action : cleanup) { + try { action.run(); } + catch (RuntimeException error) { log.warn("Fork 资源关闭失败 projectCode={} errorType={}", projectCode, error.getClass().getSimpleName()); } + } + cleanup.clear(); + } + + public static void checkpoint() { + ForkDeadline deadline = current(); + if (deadline != null) deadline.check(); + } + + public void check() { + if (cancellation != null || Thread.currentThread().isInterrupted() || System.nanoTime() >= expires) + throw failure(); + } + + public Duration remaining() { + check(); + return Duration.ofNanos(Math.max(1, expires - System.nanoTime())); + } + + public static Duration bounded(Duration maximum) { + ForkDeadline deadline = current(); + if (deadline == null) return maximum; + Duration remaining = deadline.remaining(); + return remaining.compareTo(maximum) < 0 ? remaining : maximum; + } + + public synchronized void beginWrite() { check(); writing = true; } + public synchronized void confirmWrite() { writing = false; } + synchronized void cancel(String code) { cancellation = code; } + + public BusinessException failure() { + String code = cancellation == null ? "FORK_TIMEOUT" : cancellation; + String message = "Fork 已超时或取消,请先核对工作区项目状态"; + if (writing) return new GiteaFailure(504, "REMOTE_RESULT_UNKNOWN", message, true); + return new BusinessException(code.equals("FORK_CANCELLED") ? 499 : 504, code, message, details()); + } + + public Map details() { return projectCode == null ? null : Map.of("projectCode", projectCode); } +} diff --git a/community-service/src/main/java/com/linkerbot/community/service/fork/ForkExecutor.java b/community-service/src/main/java/com/linkerbot/community/service/fork/ForkExecutor.java new file mode 100644 index 0000000..07c45a5 --- /dev/null +++ b/community-service/src/main/java/com/linkerbot/community/service/fork/ForkExecutor.java @@ -0,0 +1,90 @@ +package com.linkerbot.community.service.fork; + +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.config.ForkProperties; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import java.util.Set; +import java.util.concurrent.*; +import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.springframework.stereotype.Component; + +/** 同步等待结果,但工作线程及并发名额的生命周期不依赖 HTTP 连接。 */ +@Component +public class ForkExecutor { + private static final Logger log = LoggerFactory.getLogger(ForkExecutor.class); + private final ForkProperties properties; + private final GitSnapshotCopier copier; + private final Semaphore permits; + private final ExecutorService workers = Executors.newThreadPerTaskExecutor(Thread.ofVirtual().name("project-fork-", 0).factory()); + private final Set> active = ConcurrentHashMap.newKeySet(); + + public ForkExecutor(ForkProperties properties, GitSnapshotCopier copier) { + this.properties = properties; + this.copier = copier; + permits = new Semaphore(properties.getMaxConcurrent()); + } + + @PostConstruct + public void inspectRuntime() { + try { copier.checkAvailable(); } + catch (BusinessException error) { log.warn("Fork 运行依赖不可用 code={}", error.code()); } + } + + public T execute(Function action) { + ForkDeadline deadline = new ForkDeadline(properties.getTimeout()); + if (!permits.tryAcquire()) throw new BusinessException(429, "FORK_BUSY", "Fork 繁忙,请稍后手动重试"); + try { copier.checkAvailable(); } + catch (RuntimeException error) { permits.release(); throw error; } + String requestId = MDC.get("requestId"); + FutureTask task = new FutureTask<>(() -> { + ForkDeadline.bind(deadline); + if (requestId != null) MDC.put("requestId", requestId); + try { deadline.check(); return action.apply(deadline); } + finally { deadline.closeResources(); ForkDeadline.unbind(); MDC.remove("requestId"); } + }); + active.add(task); + try { + workers.execute(() -> { + try { task.run(); } + finally { active.remove(task); permits.release(); } + }); + } catch (RejectedExecutionException error) { + active.remove(task); + permits.release(); + throw new BusinessException(503, "FORK_UNAVAILABLE", "Fork 正在停止,请稍后重试"); + } + try { + return task.get(deadline.remaining().toNanos(), TimeUnit.NANOSECONDS); + } catch (TimeoutException error) { + deadline.cancel("FORK_TIMEOUT"); + task.cancel(true); + throw failure(deadline); + } catch (InterruptedException error) { + deadline.cancel("FORK_CANCELLED"); + task.cancel(true); + Thread.currentThread().interrupt(); + throw failure(deadline); + } catch (CancellationException error) { + throw failure(deadline); + } catch (BusinessException error) { + deadline.cancel("FORK_TIMEOUT"); + task.cancel(true); + throw failure(deadline); + } catch (ExecutionException error) { + if (error.getCause() instanceof RuntimeException cause) throw cause; + throw new BusinessException(500, "FORK_COPY_FAILED", "Fork 复制失败", deadline.details()); + } + } + + private BusinessException failure(ForkDeadline deadline) { + BusinessException error = deadline.failure(); + return new BusinessException(error.status(), error.code(), error.getMessage(), deadline.details()); + } + + @PreDestroy + public void close() { active.forEach(task -> task.cancel(true)); workers.shutdownNow(); } +} diff --git a/community-service/src/main/java/com/linkerbot/community/service/fork/GitCommandRunner.java b/community-service/src/main/java/com/linkerbot/community/service/fork/GitCommandRunner.java new file mode 100644 index 0000000..c4ef018 --- /dev/null +++ b/community-service/src/main/java/com/linkerbot/community/service/fork/GitCommandRunner.java @@ -0,0 +1,188 @@ +package com.linkerbot.community.service.fork; + +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.config.ForkProperties; +import com.linkerbot.community.service.gitea.GiteaFailure; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** 不执行 Shell,不记录参数或原始输出;输出与磁盘占用均有上限。 */ +@Component +public class GitCommandRunner { + private static final Logger log = LoggerFactory.getLogger(GitCommandRunner.class); + private static final int MAX_OUTPUT_BYTES = 8 * 1024 * 1024; + private final ForkProperties properties; + + public GitCommandRunner(ForkProperties properties) { this.properties = properties; } + + public record Result(int exitCode, String output, boolean rejected) {} + + public Result run(Path workspace, ForkDeadline deadline, String stage, Map configuration, + Map identity, String... arguments) { + deadline.check(); + List command = new ArrayList<>(); + command.add(properties.getGitExecutable()); + if (workspace != null) command.add("--git-dir=" + workspace); + command.addAll(List.of(arguments)); + ProcessBuilder builder = new ProcessBuilder(command); + Map environment = builder.environment(); + Map inherited = new HashMap<>(environment); + environment.clear(); + for (String key : List.of("PATH", "SystemRoot", "TMPDIR", "TEMP", "TMP")) + if (inherited.containsKey(key)) environment.put(key, inherited.get(key)); + environment.put("LC_ALL", "C.UTF-8"); + environment.put("GIT_CONFIG_NOSYSTEM", "1"); + environment.put("GIT_CONFIG_GLOBAL", "/dev/null"); + environment.put("GIT_ATTR_NOSYSTEM", "1"); + environment.put("GIT_TERMINAL_PROMPT", "0"); + environment.put("GIT_LFS_SKIP_SMUDGE", "1"); + Map settings = new LinkedHashMap<>(); + settings.put("core.hooksPath", "/dev/null"); + settings.put("credential.helper", ""); + settings.put("http.followRedirects", "false"); + settings.put("protocol.allow", "never"); + settings.put("protocol.http.allow", "always"); + settings.put("protocol.https.allow", "always"); + settings.putAll(configuration); + environment.put("GIT_CONFIG_COUNT", Integer.toString(settings.size())); + int index = 0; + for (Map.Entry setting : settings.entrySet()) { + environment.put("GIT_CONFIG_KEY_" + index, setting.getKey()); + environment.put("GIT_CONFIG_VALUE_" + index, setting.getValue()); + index++; + } + environment.putAll(identity); + Process process = null; + Set children = new HashSet<>(); + Thread reader = null; + Thread errorReader = null; + AtomicBoolean overflow = new AtomicBoolean(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + ByteArrayOutputStream errors = new ByteArrayOutputStream(); + long started = System.nanoTime(); + try { + process = builder.start(); + process.getOutputStream().close(); + InputStream stdout = process.getInputStream(); + reader = Thread.ofVirtual().start(() -> drain(stdout, output, overflow, MAX_OUTPUT_BYTES)); + InputStream stderr = process.getErrorStream(); + errorReader = Thread.ofVirtual().start(() -> drain(stderr, errors, new AtomicBoolean(), 32768)); + long checked = 0; + while (true) { + process.descendants().forEach(children::add); + deadline.check(); + if (overflow.get()) throw resourceLimit(); + if (workspace != null && System.nanoTime() - checked >= TimeUnit.MILLISECONDS.toNanos(250)) { + checkDisk(workspace); + checked = System.nanoTime(); + } + if (process.waitFor(50, TimeUnit.MILLISECONDS)) break; + } + while (reader.isAlive()) { reader.join(50); deadline.check(); } + while (errorReader.isAlive()) { errorReader.join(50); deadline.check(); } + deadline.check(); + if (overflow.get()) throw resourceLimit(); + if (workspace != null) checkDisk(workspace); + int exitCode = process.exitValue(); + String diagnostic = errors.toString(StandardCharsets.UTF_8).toLowerCase(Locale.ROOT); + boolean rejected = exitCode != 0 && (diagnostic.contains("authentication failed") + || diagnostic.contains("could not read username") || diagnostic.contains("[remote rejected]") + || diagnostic.contains("requested url returned error: 401") || diagnostic.contains("requested url returned error: 403") + || diagnostic.contains("requested url returned error: 404")); + log.info("Fork Git 阶段完成 projectCode={} stage={} exitCode={} failureType={} durationMs={}", + deadline.projectCode(), stage, exitCode, rejected ? "REMOTE_REJECTED" : exitCode == 0 ? "NONE" : "EXIT_FAILURE", + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started)); + return new Result(exitCode, output.toString(StandardCharsets.UTF_8).trim(), rejected); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + deadline.check(); + throw deadline.failure(); + } catch (IOException error) { + log.warn("Fork Git 进程异常 projectCode={} stage={} errorType={}", deadline.projectCode(), stage, error.getClass().getSimpleName()); + throw failed(deadline); + } finally { + if (process != null) stop(process, children); + boolean interrupted = Thread.interrupted(); + for (Thread drain : new Thread[] {reader, errorReader}) { + while (drain != null && drain.isAlive()) { + try { drain.join(); } + catch (InterruptedException error) { interrupted = true; } + } + } + if (interrupted) Thread.currentThread().interrupt(); + } + } + + private void drain(InputStream source, ByteArrayOutputStream target, AtomicBoolean overflow, int limit) { + try (InputStream input = source) { + byte[] buffer = new byte[8192]; + int count; + while ((count = input.read(buffer)) != -1) { + if (count > limit - target.size()) overflow.set(true); + if (!overflow.get()) target.write(buffer, 0, count); + } + } catch (IOException ignored) { + // 进程被终止时管道会关闭;最终由退出状态和截止时间判断结果。 + } + } + + private void stop(Process process, Set descendants) { + boolean interrupted = Thread.interrupted(); + process.descendants().forEach(descendants::add); + descendants.forEach(child -> { if (child.isAlive()) child.destroyForcibly(); }); + try { + // 先让父进程回收子进程,再终止仍未退出的父进程,避免遗留僵尸。 + try { if (!descendants.isEmpty()) process.waitFor(1, TimeUnit.SECONDS); } + catch (InterruptedException error) { interrupted = true; } + if (process.isAlive()) process.destroyForcibly(); + while (process.isAlive()) { + try { process.waitFor(100, TimeUnit.MILLISECONDS); } + catch (InterruptedException error) { interrupted = true; } + } + for (ProcessHandle child : descendants) { + while (child.isAlive()) { + try { child.onExit().get(100, TimeUnit.MILLISECONDS); } + catch (InterruptedException error) { interrupted = true; } + catch (java.util.concurrent.ExecutionException | java.util.concurrent.TimeoutException ignored) { child.destroyForcibly(); } + } + } + } finally { + if (interrupted) Thread.currentThread().interrupt(); + } + } + + public void checkDisk(Path workspace) throws IOException { + long[] bytes = {0}; + Files.walkFileTree(workspace, new SimpleFileVisitor<>() { + @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) { + long size = attributes.size(); + if (size > properties.getMaxTemporaryBytes() - bytes[0]) throw resourceLimit(); + bytes[0] += size; + return FileVisitResult.CONTINUE; + } + @Override public FileVisitResult visitFileFailed(Path file, IOException error) throws IOException { + if (error instanceof NoSuchFileException) return FileVisitResult.CONTINUE; + throw error; + } + }); + } + + private BusinessException resourceLimit() { + return new BusinessException(413, "FORK_RESOURCE_LIMIT", "Fork 临时磁盘或元数据超过限制"); + } + + static BusinessException failed(ForkDeadline deadline) { + if (deadline.writing()) return new GiteaFailure(504, "REMOTE_RESULT_UNKNOWN", "Fork 写入结果待核对,请先查看工作区", true); + return new BusinessException(502, "FORK_COPY_FAILED", "Fork 快照或 LFS 复制失败,请通过 requestId 排查"); + } +} diff --git a/community-service/src/main/java/com/linkerbot/community/service/fork/GitSnapshotCopier.java b/community-service/src/main/java/com/linkerbot/community/service/fork/GitSnapshotCopier.java new file mode 100644 index 0000000..0c7fa2d --- /dev/null +++ b/community-service/src/main/java/com/linkerbot/community/service/fork/GitSnapshotCopier.java @@ -0,0 +1,16 @@ +package com.linkerbot.community.service.fork; + +import com.linkerbot.community.service.gitea.GiteaClient.Repository; + +/** 仓库传输边界;准备阶段不创建远端仓库,业务测试可使用内存替身。 */ +public interface GitSnapshotCopier { + void checkAvailable(); + + PreparedSnapshot prepare(Repository source, String publishedSha, String authorName, String authorCode, + ForkDeadline deadline); + + interface PreparedSnapshot extends AutoCloseable { + String push(Repository target); + @Override void close(); + } +} diff --git a/community-service/src/main/java/com/linkerbot/community/service/fork/LfsEndpointGuard.java b/community-service/src/main/java/com/linkerbot/community/service/fork/LfsEndpointGuard.java new file mode 100644 index 0000000..91692cd --- /dev/null +++ b/community-service/src/main/java/com/linkerbot/community/service/fork/LfsEndpointGuard.java @@ -0,0 +1,165 @@ +package com.linkerbot.community.service.fork; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +/** git-lfs 自身不支持禁止重定向;在任务内约束所有 batch/action 请求到配置中的 Gitea。 */ +final class LfsEndpointGuard implements AutoCloseable { + private static final int MAX_METADATA = 8 * 1024 * 1024; + private final URI trustedBase; + private final URI endpoint; + private final String authorization; + private final String localAuthorization = "Bearer " + UUID.randomUUID(); + private final ForkDeadline deadline; + private final ObjectMapper json; + private final HttpServer server; + private final HttpClient http; + private final ExecutorService workers = Executors.newVirtualThreadPerTaskExecutor(); + private final ScheduledExecutorService timer = Executors.newSingleThreadScheduledExecutor(Thread.ofVirtual().factory()); + private final Map actions = new ConcurrentHashMap<>(); + private final Set exchanges = ConcurrentHashMap.newKeySet(); + private final Set responses = ConcurrentHashMap.newKeySet(); + private final AtomicBoolean closed = new AtomicBoolean(); + private record Action(URI url, Map headers) {} + + LfsEndpointGuard(URI trustedBase, URI endpoint, String authorization, ForkDeadline deadline, ObjectMapper json) throws IOException { + this.trustedBase = trustedBase; + this.endpoint = endpoint; + this.authorization = authorization; + this.deadline = deadline; + this.json = json; + if (!allowed(endpoint)) throw new IOException("LFS endpoint rejected"); + http = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).connectTimeout(deadline.remaining()).build(); + try { server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); } + catch (IOException error) { http.shutdownNow(); throw error; } + try { + server.setExecutor(workers); + server.createContext("/lfs", this::handle); + server.start(); + timer.schedule(this::close, deadline.remaining().toNanos(), TimeUnit.NANOSECONDS); + } catch (RuntimeException error) { close(); throw error; } + } + + String endpoint() { return "http://127.0.0.1:" + server.getAddress().getPort() + "/lfs"; } + String localAuthorization() { return localAuthorization; } + + private boolean allowed(URI uri) { + return uri.getRawUserInfo() == null && uri.getRawFragment() == null + && trustedBase.getScheme().equalsIgnoreCase(uri.getScheme()) + && trustedBase.getRawAuthority().equalsIgnoreCase(uri.getRawAuthority()) + && uri.normalize().getRawPath().startsWith(trustedBase.getRawPath().replaceAll("/+$", "") + "/"); + } + + private void handle(HttpExchange exchange) { + exchanges.add(exchange); + try (HttpExchange request = exchange) { + deadline.check(); + String supplied = request.getRequestHeaders().getFirst("Authorization"); + if (supplied == null || !MessageDigest.isEqual(supplied.getBytes(StandardCharsets.UTF_8), localAuthorization.getBytes(StandardCharsets.UTF_8))) { + request.sendResponseHeaders(403, -1); + return; + } + String path = request.getRequestURI().getPath(); + boolean batch = path.equals("/lfs/objects/batch") && request.getRequestMethod().equals("POST"); + Action action = batch ? new Action(URI.create(endpoint + "/objects/batch"), Map.of()) : actions.get(path); + if (action == null || !Set.of("GET", "POST", "PUT").contains(request.getRequestMethod())) { + request.sendResponseHeaders(400, -1); + return; + } + HttpRequest.Builder outbound = HttpRequest.newBuilder(action.url()).timeout(deadline.remaining()) + .header("Authorization", authorization); + if (batch) outbound.header("Accept", "application/vnd.git-lfs+json"); + action.headers().forEach(outbound::setHeader); + String type = request.getRequestHeaders().getFirst("Content-Type"); + if (type != null) outbound.setHeader("Content-Type", type); + HttpRequest.BodyPublisher body = batch ? HttpRequest.BodyPublishers.ofByteArray(metadata(request.getRequestBody())) + : request.getRequestMethod().equals("GET") ? HttpRequest.BodyPublishers.noBody() + : HttpRequest.BodyPublishers.ofInputStream(request::getRequestBody); + outbound.method(request.getRequestMethod(), body); + HttpResponse response = http.send(outbound.build(), HttpResponse.BodyHandlers.ofInputStream()); + InputStream stream = response.body(); + responses.add(stream); + try (InputStream input = stream) { + // 不把 Location 传给 git-lfs,禁止它自行追随重定向。 + if (response.statusCode() < 200 || response.statusCode() >= 300) { + request.sendResponseHeaders(response.statusCode() >= 300 && response.statusCode() < 400 ? 502 : response.statusCode(), -1); + return; + } + if (batch) { + JsonNode result = json.readTree(metadata(input)); + for (JsonNode object : result.path("objects")) { + JsonNode entries = object.path("actions"); + for (JsonNode value : entries) { + if (!(value instanceof ObjectNode item)) throw new IOException("LFS action rejected"); + URI destination = URI.create(item.path("href").asText()); + if (!allowed(destination)) throw new IOException("LFS action origin rejected"); + Map headers = new java.util.HashMap<>(); + JsonNode header = item.path("header"); + header.fields().forEachRemaining(field -> { + if (Set.of("authorization", "content-type", "accept").contains(field.getKey().toLowerCase(java.util.Locale.ROOT))) + headers.put(field.getKey(), field.getValue().asText()); + }); + String route = "/lfs/action/" + UUID.randomUUID(); + actions.put(route, new Action(destination, headers)); + item.put("href", endpoint().replace("/lfs", "") + route); + item.putObject("header").put("Authorization", localAuthorization); + } + } + byte[] bytes = json.writeValueAsBytes(result); + request.getResponseHeaders().set("Content-Type", "application/vnd.git-lfs+json"); + request.sendResponseHeaders(response.statusCode(), bytes.length); + request.getResponseBody().write(bytes); + } else { + request.sendResponseHeaders(response.statusCode(), 0); + byte[] buffer = new byte[65536]; + int count; + while ((count = input.read(buffer)) != -1) { + deadline.check(); + request.getResponseBody().write(buffer, 0, count); + } + } + } finally { responses.remove(stream); } + } catch (IOException | InterruptedException | RuntimeException error) { + if (error instanceof InterruptedException) Thread.currentThread().interrupt(); + // 不输出服务端原始响应、地址、头或凭据;git-lfs 会以非零状态结束该阶段。 + exchange.close(); + } finally { exchanges.remove(exchange); } + } + + private byte[] metadata(InputStream input) throws IOException { + byte[] bytes = input.readNBytes(MAX_METADATA + 1); + if (bytes.length > MAX_METADATA) throw new IOException("LFS metadata limit exceeded"); + return bytes; + } + + @Override public void close() { + if (closed.compareAndSet(false, true)) { + server.stop(0); + exchanges.forEach(HttpExchange::close); + responses.forEach(stream -> { try { stream.close(); } catch (IOException ignored) {} }); + http.shutdownNow(); + workers.shutdownNow(); + timer.shutdownNow(); + actions.clear(); + } + // 并发名额只能在转发流也实际退出后释放;不能只停止命令行进程。 + workers.close(); + } +} diff --git a/community-service/src/main/java/com/linkerbot/community/service/gitea/GiteaClient.java b/community-service/src/main/java/com/linkerbot/community/service/gitea/GiteaClient.java index 0b11ee7..daa4136 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/gitea/GiteaClient.java +++ b/community-service/src/main/java/com/linkerbot/community/service/gitea/GiteaClient.java @@ -7,6 +7,8 @@ import java.util.List; import java.time.Duration; public interface GiteaClient { + default String authenticatedUsername() { throw new UnsupportedOperationException("未实现服务账号查询"); } + record Repository(long id, String owner, String name) {} record StoredFile(String path, String blobSha, byte[] bytes) {} @@ -33,7 +35,7 @@ public interface GiteaClient { default void checkMigrationSettings() { throw new UnsupportedOperationException("未实现迁移能力检查"); } default List migrationWarnings(Repository repository, String sha, long maxFileBytes) { - return List.of("社区下载仍受单文件限制,LFS 实体读取及特殊 Git 文件的 Fork 能力未扩展"); + return List.of("在线下载仍受单文件限制,LFS 实体在线读取能力未扩展;Git 快照 Fork 保留子模块引用,不递归复制依赖"); } Repository createRepository(String name, boolean initialize); diff --git a/community-service/src/main/java/com/linkerbot/community/service/gitea/RestGiteaClient.java b/community-service/src/main/java/com/linkerbot/community/service/gitea/RestGiteaClient.java index 7a495bc..fefca47 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/gitea/RestGiteaClient.java +++ b/community-service/src/main/java/com/linkerbot/community/service/gitea/RestGiteaClient.java @@ -7,6 +7,7 @@ import com.linkerbot.community.common.BusinessException; import com.linkerbot.community.common.FilePaths; import com.linkerbot.community.common.PageResponse; import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.fork.ForkDeadline; import org.springframework.stereotype.Component; @@ -62,7 +63,7 @@ public class RestGiteaClient implements GiteaClient { String base = properties.getGitea().getBaseUrl().replaceAll("/+$", ""); HttpRequest.Builder builder = HttpRequest.newBuilder(URI.create(base + "/api/v1" + path)) - .timeout(timeout) + .timeout(ForkDeadline.bounded(timeout)) .header("Accept", "application/json") .header("Authorization", "token " + properties.getGitea().getToken()); if (body == null) builder.method(method, HttpRequest.BodyPublishers.noBody()); @@ -106,6 +107,14 @@ public class RestGiteaClient implements GiteaClient { } } + @Override + public String authenticatedUsername() { + JsonNode user = parse(request("GET", "/user", null).body(), false); + if (!user.path("login").isTextual() || user.path("login").asText().isBlank()) + throw new GiteaFailure(502, "FORK_COPY_FAILED", "无法确认 Gitea 服务账号身份", false); + return user.path("login").asText(); + } + @Override public Repository createRepository(String name, boolean initialize) { Map body = new LinkedHashMap<>(); @@ -186,7 +195,7 @@ public class RestGiteaClient implements GiteaClient { @Override public List migrationWarnings(Repository repo, String sha, long maxFileBytes) { List warnings = new ArrayList<>(); - warnings.add("已请求迁移 LFS;社区 API 的 LFS 实体读取能力未扩展"); + warnings.add("已请求迁移 LFS;在线 API 的 LFS 实体读取能力未扩展,Git 快照 Fork 会复制发布版本引用的实体"); long large = 0; long links = 0; long modules = 0; @@ -206,8 +215,8 @@ public class RestGiteaClient implements GiteaClient { if (entries.isEmpty() || page == 10000) { warnings.add("文件树扫描不完整,需要人工核对剩余文件"); break; } } if (large > 0) warnings.add("发布快照有 " + large + " 个文件超过社区单文件读取限制 " + maxFileBytes + " 字节"); - if (links > 0) warnings.add("发布快照有 " + links + " 个符号链接,社区普通文件读取和 Fork 不支持"); - if (modules > 0) warnings.add("发布快照有 " + modules + " 个子模块引用,未递归迁移且社区 Fork 不支持"); + if (links > 0) warnings.add("发布快照有 " + links + " 个符号链接,在线普通文件读取不支持,Git 快照 Fork 保留链接"); + if (modules > 0) warnings.add("发布快照有 " + modules + " 个子模块引用,Git 快照 Fork 保留引用,不递归复制依赖仓库"); return List.copyOf(warnings); } diff --git a/community-service/src/main/java/com/linkerbot/community/service/impl/AiSearchServiceImpl.java b/community-service/src/main/java/com/linkerbot/community/service/impl/AiSearchServiceImpl.java index 5412100..40f026f 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/impl/AiSearchServiceImpl.java +++ b/community-service/src/main/java/com/linkerbot/community/service/impl/AiSearchServiceImpl.java @@ -8,6 +8,7 @@ import com.linkerbot.community.common.BusinessException; import com.linkerbot.community.service.AiSearchService; import com.linkerbot.community.service.ai.AiProjectAssistant; import com.linkerbot.community.service.config.AiProperties; +import com.linkerbot.community.service.config.ForkProperties; import jakarta.annotation.PreDestroy; import java.io.IOException; import java.time.Duration; @@ -26,6 +27,7 @@ public class AiSearchServiceImpl implements AiSearchService { private static final Logger log = LoggerFactory.getLogger(AiSearchServiceImpl.class); private final AiProperties properties; private final AiProjectAssistant assistant; + private final ForkProperties forkProperties; private final Semaphore permits; private final ExecutorService workers = Executors.newThreadPerTaskExecutor( Thread.ofVirtual().name("ai-search-", 0).factory()); @@ -35,9 +37,10 @@ public class AiSearchServiceImpl implements AiSearchService { .expireAfterAccess(Duration.ofMinutes(1)).maximumSize(10000).build(); private final Set active = ConcurrentHashMap.newKeySet(); - public AiSearchServiceImpl(AiProperties properties, AiProjectAssistant assistant) { + public AiSearchServiceImpl(AiProperties properties, AiProjectAssistant assistant, ForkProperties forkProperties) { this.properties = properties; this.assistant = assistant; + this.forkProperties = forkProperties; permits = new Semaphore(properties.getMaxConcurrent()); } @@ -97,7 +100,7 @@ public class AiSearchServiceImpl implements AiSearchService { private final ChatRequest request; private final String requestId; private final long started = System.nanoTime(); - private final SseEmitter emitter = new SseEmitter(properties.getSearchTimeout().toMillis()); + private final SseEmitter emitter = new SseEmitter(properties.getSearchTimeout().plus(forkProperties.getTimeout()).plusSeconds(15).toMillis()); private final AtomicBoolean finished = new AtomicBoolean(); private final AtomicBoolean released = new AtomicBoolean(); private final FutureTask task; @@ -152,6 +155,11 @@ public class AiSearchServiceImpl implements AiSearchService { private void send(String event, Object data) { if (finished.get() || Thread.currentThread().isInterrupted()) throw cancelled(); + if (data instanceof Progress progress && "forking".equals(progress.stage())) { + if (deadline != null) deadline.cancel(false); + deadline = timer.schedule(() -> fail(new BusinessException(504, "FORK_TIMEOUT", "Fork 超时,请先核对工作区项目状态")), + forkProperties.getTimeout().plusSeconds(15).toMillis(), TimeUnit.MILLISECONDS); + } if (!write(event, success(data))) throw cancelled(); } diff --git a/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java b/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java index 1e1fe4a..8860899 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java +++ b/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java @@ -13,6 +13,9 @@ import com.linkerbot.community.service.ProjectService; import com.linkerbot.community.service.UserService; import com.linkerbot.community.service.config.CommunityProperties; import com.linkerbot.community.service.gitea.*; +import com.linkerbot.community.service.fork.ForkDeadline; +import com.linkerbot.community.service.fork.ForkExecutor; +import com.linkerbot.community.service.fork.GitSnapshotCopier; import com.linkerbot.community.service.mapper.*; import com.linkerbot.community.service.oss.UrdfResources; import com.linkerbot.community.service.support.ProjectLocks; @@ -42,6 +45,8 @@ public class ProjectServiceImpl implements ProjectService { private final ProjectLocks locks; // 离线导入只加载仓库和数据库组件,没有 URDF 时无需初始化 OSS 模块。 private final ObjectProvider urdf; + private final ObjectProvider forks; + private final ObjectProvider snapshots; @Override public List types() { @@ -180,6 +185,10 @@ public class ProjectServiceImpl implements ProjectService { } private Project initialize(long userId, ProjectInput input, Project source) { + return initialize(userId, input, source, null); + } + + private Project initialize(long userId, ProjectInput input, Project source, ForkDeadline deadline) { validateInput(input); Project project = new Project(); project.setProjectCode(ProjectCodes.next()); @@ -201,6 +210,10 @@ public class ProjectServiceImpl implements ProjectService { } for (int attempt = 1; attempt <= 5; attempt++) { try { + if (deadline != null) { + deadline.check(); + deadline.projectCode(project.getProjectCode()); + } projects.insert(project); return project; } catch (DuplicateKeyException duplicate) { @@ -226,7 +239,7 @@ public class ProjectServiceImpl implements ProjectService { public ProjectView create(long userId, ProjectInput input) { users.require(userId); Project project = initialize(userId, input, null); - return locks.write(project.getProjectCode(), () -> createRemote(project, null)); + return locks.write(project.getProjectCode(), () -> createRemote(project)); } @Override @@ -238,33 +251,97 @@ public class ProjectServiceImpl implements ProjectService { ProjectInput data = new ProjectInput( input.name(), source.getDescription(), source.getProjectType(), source.getCoverUrl()); - Project project = initialize(userId, data, source); - return locks.write( - project.getProjectCode(), - () -> createRemote(project, new ForkContext(source, user))); + validateInput(data); + ForkExecutor executor = forks.getIfAvailable(); + if (executor == null) throw new BusinessException(503, "FORK_UNAVAILABLE", "当前进程未开启 Fork"); + return executor.execute(deadline -> { + deadline.check(); + Project project; + try { project = initialize(userId, data, source, deadline); } + catch (BusinessException error) { throw error; } + catch (RuntimeException error) { + if (deadline.projectCode() != null) locks.markUncertain(deadline.projectCode()); + log.error("Fork 项目记录写入异常 projectCode={} sourceProjectCode={} errorType={}", + deadline.projectCode(), sourceCode, error.getClass().getSimpleName()); + throw new BusinessException(500, "PROJECT_INITIALIZATION_ERROR", "Fork 项目记录待核对,请按项目编码人工检查", deadline.details()); + } + return locks.write(project.getProjectCode(), () -> forkRemote(project, source, user, deadline)); + }); } - private record ForkContext(Project source, User user) {} + private ProjectView forkRemote(Project project, Project source, User user, ForkDeadline deadline) { + long started = System.nanoTime(); + log.info("Fork 开始 projectCode={} sourceProjectCode={} sourceCommitSha={}", + project.getProjectCode(), source.getProjectCode(), project.getSourceCommitSha()); + try (GitSnapshotCopier.PreparedSnapshot snapshot = snapshots.getObject().prepare( + repository(source), project.getSourceCommitSha(), user.getNickname(), user.getUserCode(), deadline)) { + deadline.beginWrite(); + GiteaClient.Repository repo; + try { + repo = gitea.createRepository(project.getGiteaRepoName(), false); + } catch (GiteaFailure error) { + if (!error.uncertain()) deadline.confirmWrite(); + throw error; + } + project.setGiteaRepoId(repo.id()); + // 先保存仓库关联,再执行 Git/LFS 推送,便于中断后按项目编码核对。 + if (projects.updateById(project) != 1) throw new IllegalStateException("Fork 仓库关联回写失败"); + deadline.confirmWrite(); + deadline.check(); + String commit = snapshot.push(repo); + deadline.check(); + if (source.getUrdfMainObjectKey() != null && !source.getUrdfMainObjectKey().isEmpty()) + urdf.getObject().copy(source, project); + deadline.check(); + project.setRepoStatus("READY"); + project.setUpdatedAt(now()); + if (projects.updateById(project) != 1) throw new IllegalStateException("Fork 完成状态回写失败"); + log.info("Fork 完成 projectCode={} sourceProjectCode={} sourceCommitSha={} snapshotCommitSha={} durationMs={}", + project.getProjectCode(), source.getProjectCode(), project.getSourceCommitSha(), commit, + java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started)); + return view(project, project.getOwnerUserId()); + } catch (RuntimeException error) { + boolean uncertain = deadline.writing() || !(error instanceof BusinessException) + || error instanceof GiteaFailure remote && remote.uncertain(); + String code = error instanceof BusinessException business ? business.code() : "PROJECT_INITIALIZATION_ERROR"; + int status = error instanceof BusinessException business ? business.status() : 500; + String message = error instanceof BusinessException business ? business.getMessage() : "Fork 初始化未完成,请按项目编码核对"; + if (deadline.writing() && error instanceof BusinessException) { + code = "REMOTE_RESULT_UNKNOWN"; + status = 504; + message = "Fork 远端写入结果待核对,请先查看工作区项目状态"; + } + boolean interrupted = Thread.interrupted(); + try { + project.setRepoStatus(uncertain ? "CREATING" : "FAILED"); + project.setUpdatedAt(now()); + if (uncertain) locks.markUncertain(project.getProjectCode()); + if (projects.updateById(project) != 1) throw new IllegalStateException("Fork 失败状态回写失败"); + } catch (RuntimeException persistence) { + locks.markUncertain(project.getProjectCode()); + log.error("Fork 状态回写异常 projectCode={} errorType={}", project.getProjectCode(), persistence.getClass().getSimpleName()); + code = "PROJECT_INITIALIZATION_ERROR"; + status = 500; + message = "Fork 状态未确认,请按项目编码人工核对"; + } finally { + if (interrupted) Thread.currentThread().interrupt(); + } + log.warn("Fork 失败 projectCode={} sourceProjectCode={} sourceCommitSha={} code={} uncertain={} errorType={} durationMs={}", + project.getProjectCode(), source.getProjectCode(), project.getSourceCommitSha(), code, uncertain, + error.getClass().getSimpleName(), java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started)); + throw new BusinessException(status, code, message, Map.of("projectCode", project.getProjectCode())); + } + } - private ProjectView createRemote(Project project, ForkContext fork) { + private ProjectView createRemote(Project project) { try { GiteaClient.Repository repo = - gitea.createRepository(project.getGiteaRepoName(), fork == null); + gitea.createRepository(project.getGiteaRepoName(), true); project.setGiteaRepoId(repo.id()); - if (fork != null) { - copySnapshot( - repository(fork.source()), - fork.source().getPublishedCommitSha(), - repo, - fork.user()); - if (fork.source().getUrdfMainObjectKey() != null - && !fork.source().getUrdfMainObjectKey().isEmpty()) - urdf.getObject().copy(fork.source(), project); - } project.setRepoStatus("READY"); project.setUpdatedAt(now()); projects.updateById(project); - log.info("仓库创建完成 projectCode={} fork={}", project.getProjectCode(), fork != null); + log.info("仓库创建完成 projectCode={} fork={}", project.getProjectCode(), false); return view(project, project.getOwnerUserId()); } catch (GiteaFailure error) { project.setRepoStatus(error.uncertain() ? "CREATING" : "FAILED"); @@ -304,48 +381,6 @@ public class ProjectServiceImpl implements ProjectService { } } - private void copySnapshot( - GiteaClient.Repository source, String sha, GiteaClient.Repository target, User user) { - Deque directories = new ArrayDeque<>(); - directories.add(""); - List batch = new ArrayList<>(); - long batchBytes = 0; - while (!directories.isEmpty()) { - for (TreeEntry entry : gitea.tree(source, sha, directories.removeFirst())) { - FilePaths.file(entry.path()); - if (entry.type().equals("dir")) { - directories.add(entry.path()); - continue; - } - if (!entry.type().equals("file")) - throw BusinessException.badRequest("Fork 暂不支持符号链接或子模块"); - GiteaClient.StoredFile file = gitea.file(source, sha, entry.path()); - if (!batch.isEmpty() - && (batchBytes + file.bytes().length > config.getMaxRequestBytes() - || batch.size() >= 100)) { - gitea.commit( - target, - batch, - "Fork community snapshot", - user.getNickname(), - user.getUserCode()); - batch.clear(); - batchBytes = 0; - } - batch.add(new GiteaClient.Change("create", file.path(), null, file.bytes())); - batchBytes += file.bytes().length; - } - } - if (!batch.isEmpty()) - gitea.commit( - target, - batch, - "Fork community snapshot", - user.getNickname(), - user.getUserCode()); - else if (batchBytes == 0) throw BusinessException.badRequest("不能 Fork 空仓库"); - } - @Override public T write(long userId, String code, Function action) { return locks.write( diff --git a/community-service/src/main/java/com/linkerbot/community/service/oss/AliyunOssObjectStore.java b/community-service/src/main/java/com/linkerbot/community/service/oss/AliyunOssObjectStore.java index 8aa3e69..3ecdf0b 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/oss/AliyunOssObjectStore.java +++ b/community-service/src/main/java/com/linkerbot/community/service/oss/AliyunOssObjectStore.java @@ -4,6 +4,7 @@ import com.aliyun.oss.ClientBuilderConfiguration; import com.aliyun.oss.ClientException; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.OSSClient; import com.aliyun.oss.OSSException; import com.aliyun.oss.common.comm.SignVersion; import com.aliyun.oss.common.auth.DefaultCredentialProvider; @@ -11,6 +12,8 @@ import com.aliyun.oss.model.CopyObjectRequest; import com.aliyun.oss.model.ObjectMetadata; import com.linkerbot.community.common.BusinessException; import com.linkerbot.community.service.config.OssProperties; +import com.linkerbot.community.service.fork.ForkDeadline; +import com.linkerbot.community.service.gitea.GiteaFailure; import jakarta.annotation.PreDestroy; import java.util.List; import lombok.RequiredArgsConstructor; @@ -24,25 +27,54 @@ public class AliyunOssObjectStore implements OssObjectStore { private static final Logger log = LoggerFactory.getLogger(AliyunOssObjectStore.class); private final OssProperties properties; private OSS client; + private final ThreadLocal forkClients = new ThreadLocal<>(); private synchronized OSS client() { if (!properties.isEnabled()) throw new BusinessException(503, "OSS_UNAVAILABLE", "OSS URDF 上传暂未开启"); + ForkDeadline deadline = ForkDeadline.current(); + if (deadline != null) { + deadline.check(); + OSSClient bounded = forkClients.get(); + if (bounded == null) { + ClientBuilderConfiguration configuration = configuration(); + configuration.setRequestTimeoutEnabled(true); + configuration.setMaxErrorRetry(0); + bounded = (OSSClient) build(configuration); + forkClients.set(bounded); + OSSClient owned = bounded; + deadline.onClose(() -> { forkClients.remove(); owned.shutdown(); }); + } + int remaining = (int) Math.min(30000, Math.max(1, deadline.remaining().toMillis())); + bounded.getClientConfiguration().setRequestTimeout(remaining); + bounded.getClientConfiguration().setSocketTimeout(remaining); + bounded.getClientConfiguration().setConnectionTimeout(Math.min(5000, remaining)); + bounded.getClientConfiguration().setConnectionRequestTimeout(Math.min(5000, remaining)); + return bounded; + } if (client == null) { - ClientBuilderConfiguration configuration = new ClientBuilderConfiguration(); - configuration.setSignatureVersion(SignVersion.V4); - configuration.setConnectionTimeout(5000); - configuration.setSocketTimeout(30000); - configuration.setMaxErrorRetry(1); - client = OSSClientBuilder.create() + client = build(configuration()); + } + return client; + } + + private ClientBuilderConfiguration configuration() { + ClientBuilderConfiguration configuration = new ClientBuilderConfiguration(); + configuration.setSignatureVersion(SignVersion.V4); + configuration.setConnectionTimeout(5000); + configuration.setSocketTimeout(30000); + configuration.setMaxErrorRetry(1); + return configuration; + } + + private OSS build(ClientBuilderConfiguration configuration) { + return OSSClientBuilder.create() .endpoint(properties.resolvedEndpoint()) .region(properties.getRegion()) .credentialsProvider(new DefaultCredentialProvider( properties.getAccessKeyId(), properties.getAccessKeySecret())) .clientConfiguration(configuration) .build(); - } - return client; } @Override @@ -53,20 +85,29 @@ public class AliyunOssObjectStore implements OssObjectStore { } catch (OSSException error) { throw failure("head", objectKey, error); } catch (ClientException error) { + ForkDeadline.checkpoint(); throw transportFailure("head", objectKey, error); } } @Override public void copy(String sourceKey, String targetKey, String sourceEtag) { + ForkDeadline deadline = ForkDeadline.current(); + if (deadline != null) deadline.beginWrite(); try { CopyObjectRequest request = new CopyObjectRequest( properties.getBucket(), sourceKey, properties.getBucket(), targetKey); request.setMatchingETagConstraints(List.of(sourceEtag)); client().copyObject(request); + if (deadline != null) deadline.confirmWrite(); } catch (OSSException error) { + if (deadline != null) deadline.confirmWrite(); throw failure("copy", targetKey, error); } catch (ClientException error) { + if (deadline != null) { + deadline.check(); + throw new GiteaFailure(504, "REMOTE_RESULT_UNKNOWN", "Fork OSS 复制结果待核对", true); + } throw transportFailure("copy", targetKey, error); } } diff --git a/community-service/src/main/java/com/linkerbot/community/service/oss/UrdfResources.java b/community-service/src/main/java/com/linkerbot/community/service/oss/UrdfResources.java index 92e853e..4291cc2 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/oss/UrdfResources.java +++ b/community-service/src/main/java/com/linkerbot/community/service/oss/UrdfResources.java @@ -12,6 +12,7 @@ import com.linkerbot.community.common.Codes; import com.linkerbot.community.common.FilePaths; import com.linkerbot.community.service.config.OssProperties; import com.linkerbot.community.service.config.UrdfProperties; +import com.linkerbot.community.service.fork.ForkDeadline; import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; import java.util.ArrayList; @@ -114,6 +115,7 @@ public class UrdfResources { } private OssObjectStore.Metadata checkedMetadata(String prefix, UrdfFile file) { + ForkDeadline.checkpoint(); OssObjectStore.Metadata metadata = store.head(prefix + file.relativePath()); if (metadata.sizeBytes() != file.sizeBytes()) { log.warn("URDF 文件大小不符 objectKey={} expectedBytes={} actualBytes={}", @@ -156,8 +158,10 @@ public class UrdfResources { source.getProjectCode(), target.getProjectCode(), files.size()); for (UrdfFile file : files) { OssObjectStore.Metadata metadata = checkedMetadata(sourcePrefix, file); + ForkDeadline.checkpoint(); store.copy(sourcePrefix + file.relativePath(), targetPrefix + file.relativePath(), metadata.etag()); } + ForkDeadline.checkpoint(); target.setUrdfPrefix(targetPrefix); target.setUrdfMainObjectKey(targetPrefix + main); target.setUrdfFiles(manifest(files)); diff --git a/community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java b/community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java index c2dca69..2fd9ce0 100644 --- a/community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java +++ b/community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java @@ -11,6 +11,8 @@ import com.linkerbot.community.common.BusinessException; import com.linkerbot.community.common.ProjectCodes; import com.linkerbot.community.service.config.CommunityProperties; import com.linkerbot.community.service.gitea.GiteaClient; +import com.linkerbot.community.service.fork.ForkExecutor; +import com.linkerbot.community.service.fork.GitSnapshotCopier; import com.linkerbot.community.service.impl.ProjectServiceImpl; import com.linkerbot.community.service.mapper.*; import com.linkerbot.community.service.oss.UrdfResources; @@ -87,6 +89,8 @@ class ProjectCodeAllocationTest { gitea, new CommunityProperties(), new ProjectLocks(), - new StaticListableBeanFactory().getBeanProvider(UrdfResources.class)); + new StaticListableBeanFactory().getBeanProvider(UrdfResources.class), + new StaticListableBeanFactory().getBeanProvider(ForkExecutor.class), + new StaticListableBeanFactory().getBeanProvider(GitSnapshotCopier.class)); } } diff --git a/community-service/src/test/java/com/linkerbot/community/service/fork/CliGitSnapshotCopierTest.java b/community-service/src/test/java/com/linkerbot/community/service/fork/CliGitSnapshotCopierTest.java new file mode 100644 index 0000000..0e0de4f --- /dev/null +++ b/community-service/src/test/java/com/linkerbot/community/service/fork/CliGitSnapshotCopierTest.java @@ -0,0 +1,333 @@ +package com.linkerbot.community.service.fork; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.config.ForkProperties; +import com.linkerbot.community.service.gitea.GiteaClient; +import com.linkerbot.community.service.gitea.GiteaClient.Repository; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.security.MessageDigest; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class CliGitSnapshotCopierTest { + @TempDir Path root; + private final ObjectMapper json = new ObjectMapper(); + private final Map> objects = new ConcurrentHashMap<>(); + private final AtomicInteger unexpectedRequests = new AtomicInteger(); + private HttpServer lfs; + private String origin; + private ForkProperties properties; + private CommunityProperties community; + private GitCommandRunner commands; + private GiteaClient gitea; + private CliGitSnapshotCopier copier; + private volatile boolean rejectTargetDownload; + private volatile boolean redirectLfs; + private volatile boolean externalAction; + private volatile boolean rejectGitPush; + + @BeforeEach void setup() throws Exception { + lfs = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + lfs.setExecutor(Executors.newVirtualThreadPerTaskExecutor()); + origin = "http://127.0.0.1:" + lfs.getAddress().getPort(); + lfs.createContext("/", this::serveLfs); + lfs.start(); + properties = new ForkProperties(); + properties.setTemporaryDirectory(root.resolve("jobs").toString()); + community = new CommunityProperties(); + community.getGitea().setBaseUrl(origin); + community.getGitea().setToken("fixture-token"); + commands = new GitCommandRunner(properties); + gitea = mock(GiteaClient.class); + when(gitea.authenticatedUsername()).thenReturn("fixture"); + copier = new CliGitSnapshotCopier(properties, community, commands, gitea, json) { + @Override protected void verifyRemote(Repository target, String commit, ForkDeadline deadline) {} + @Override protected String repositoryUrl(Repository repo) { + return rejectGitPush && repo.name().equals("target") ? origin + "/denied.git" : repository(repo.name()).toUri().toString(); + } + @Override protected Map configuration(String endpoint) { + String name = endpoint.contains("source.git") ? "source" : "target"; + Map settings = super.configuration(origin + "/" + name + "/info/lfs"); + settings.put("protocol.file.allow", "always"); + return settings; + } + }; + copier.checkAvailable(); + git(repository("source"), null, "init", "--bare", "--quiet"); + git(repository("target"), null, "init", "--bare", "--quiet"); + } + + @AfterEach void stop() { if (lfs != null) lfs.stop(0); } + + private Path repository(String name) { return root.resolve(name + ".git"); } + private ForkDeadline deadline() { + ForkDeadline deadline = new ForkDeadline(Duration.ofSeconds(25)); + deadline.projectCode("111111111111"); + return deadline; + } + + private String git(Path repository, byte[] input, String... args) throws Exception { + List command = new ArrayList<>(List.of("git", "--git-dir=" + repository)); + command.addAll(List.of(args)); + ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().putAll(Map.of("GIT_CONFIG_NOSYSTEM", "1", "GIT_CONFIG_GLOBAL", "/dev/null", + "GIT_AUTHOR_NAME", "Fixture", "GIT_AUTHOR_EMAIL", "fixture@users.community.invalid", + "GIT_COMMITTER_NAME", "Fixture", "GIT_COMMITTER_EMAIL", "fixture@users.community.invalid")); + Process process = builder.start(); + if (input != null) process.getOutputStream().write(input); + process.getOutputStream().close(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8).trim(); + String error = new String(process.getErrorStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.waitFor(15, TimeUnit.SECONDS)).as("fixture Git completed").isTrue(); + assertThat(process.exitValue()).as("fixture Git: %s", error).isZero(); + return output; + } + + private String blob(byte[] bytes) throws Exception { return git(repository("source"), bytes, "hash-object", "-w", "--stdin"); } + private String tree(String entries) throws Exception { return git(repository("source"), entries.getBytes(StandardCharsets.UTF_8), "mktree"); } + private String commit(String tree, String parent, String message) throws Exception { + return parent == null ? git(repository("source"), null, "commit-tree", tree, "-m", message) + : git(repository("source"), null, "commit-tree", tree, "-p", parent, "-m", message); + } + private String simplePublication() throws Exception { + String tree = tree("100644 blob " + blob("published".getBytes(StandardCharsets.UTF_8)) + "\tREADME.md\n"); + String sha = commit(tree, null, "published"); + git(repository("source"), null, "update-ref", "refs/heads/main", sha); + return sha; + } + private void assertClean() throws IOException { + try (java.util.stream.Stream paths = Files.list(root.resolve("jobs"))) { assertThat(paths.toList()).isEmpty(); } + } + + @Test void preservesPublishedTreeLargeFilesModesAndGitlinksWithoutHistory() throws Exception { + String initial = simplePublication(); + String normal = blob("published".getBytes(StandardCharsets.UTF_8)); + String large = blob(new byte[11 * 1024 * 1024]); + String link = blob("README.md".getBytes(StandardCharsets.UTF_8)); + String subtree = tree("160000 commit " + "a".repeat(40) + "\tdrive_base\n"); + String publishedTree = tree("100644 blob " + normal + "\tREADME.md\n100644 blob " + normal + + "\t说明.txt\n100755 blob " + normal + "\trun.sh\n100644 blob " + large + + "\tassembly.SLDASM\n120000 blob " + link + "\treadme-link\n040000 tree " + subtree + "\turos\n"); + String published = commit(publishedTree, initial, "published special files"); + String changedTree = tree("100644 blob " + blob("unpublished private content".getBytes(StandardCharsets.UTF_8)) + "\tprivate.txt\n"); + String unpublished = commit(changedTree, published, "not published"); + git(repository("source"), null, "update-ref", "refs/heads/main", unpublished); + git(repository("source"), null, "update-ref", "refs/tags/old", initial); + String result; + try (GitSnapshotCopier.PreparedSnapshot snapshot = copier.prepare(new Repository(1, "community", "source"), published, + "测试用户", "public-code", deadline())) { + try (java.util.stream.Stream paths = Files.list(root.resolve("jobs"))) { + String config = Files.readString(paths.findFirst().orElseThrow().resolve("config")); + assertThat(config).doesNotContain("fixture-token", "Authorization", "Basic "); + } + result = snapshot.push(new Repository(2, "community", "target")); + } + assertThat(git(repository("target"), null, "rev-parse", "main^{tree}")).isEqualTo(publishedTree); + assertThat(repository("target").resolve("shallow")).doesNotExist(); + assertThat(git(repository("target"), null, "rev-list", "--parents", "main")).isEqualTo(result); + assertThat(git(repository("target"), null, "for-each-ref", "--format=%(refname)")).isEqualTo("refs/heads/main"); + assertThat(git(repository("target"), null, "ls-tree", "-r", "main")).contains("100755 blob", "120000 blob", "160000 commit"); + assertThat(git(repository("target"), null, "log", "--format=%ae", "main")).isEqualTo("public-code@users.community.invalid"); + for (String original : List.of(initial, published, unpublished)) { + Process check = new ProcessBuilder("git", "--git-dir=" + repository("target"), "cat-file", "-e", original) + .redirectError(ProcessBuilder.Redirect.DISCARD).start(); + assertThat(check.waitFor()).isNotZero(); + } + assertClean(); + } + + @Test void supportsIndependentRepeatedCopiesOfOneSource() throws Exception { + String sha = simplePublication(); + for (String target : List.of("target", "another")) { + git(repository(target), null, "init", "--bare", "--quiet"); + try (GitSnapshotCopier.PreparedSnapshot snapshot = copier.prepare(new Repository(1, "community", "source"), sha, + target, target, deadline())) { + snapshot.push(new Repository(2, "community", target)); + } + assertThat(git(repository(target), null, "rev-list", "--count", "main")).isEqualTo("1"); + } + assertClean(); + } + + private String lfsPublication(byte[] bytes, boolean storeSource) throws Exception { + String oid = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); + if (storeSource) objects.computeIfAbsent("source", ignored -> new ConcurrentHashMap<>()).put(oid, bytes); + String pointer = "version https://git-lfs.github.com/spec/v1\noid sha256:" + oid + "\nsize " + bytes.length + "\n"; + String attributes = blob("big.bin filter=lfs diff=lfs merge=lfs -text\n".getBytes(StandardCharsets.UTF_8)); + String malicious = blob(("[lfs]\nurl = " + origin + "/forbidden\nfetchinclude = no-such-file\nfetchexclude = *\n").getBytes(StandardCharsets.UTF_8)); + String tree = tree("100644 blob " + attributes + "\t.gitattributes\n100644 blob " + malicious + "\t.lfsconfig\n100644 blob " + + blob(pointer.getBytes(StandardCharsets.UTF_8)) + "\tbig.bin\n"); + String sha = commit(tree, null, "LFS publication"); + git(repository("source"), null, "update-ref", "refs/heads/main", sha); + return sha; + } + + @Test void transfersLfsEntitiesAndVerifiesDownloadFromTargetIgnoringSourceLfsconfig() throws Exception { + byte[] bytes = new byte[128 * 1024]; + new Random(42).nextBytes(bytes); + String sha = lfsPublication(bytes, true); + try (GitSnapshotCopier.PreparedSnapshot snapshot = copier.prepare(new Repository(1, "community", "source"), sha, + "测试用户", "public-code", deadline())) { + snapshot.push(new Repository(2, "community", "target")); + } + assertThat(objects.get("target").values()).singleElement().satisfies(actual -> assertThat(actual).isEqualTo(bytes)); + assertThat(unexpectedRequests).hasValue(0); + assertThat(git(repository("target"), null, "rev-parse", "main^{tree}")) + .isEqualTo(git(repository("source"), null, "rev-parse", sha + "^{tree}")); + assertClean(); + } + + @Test void refusesMissingLfsObjectBeforeAnyTargetPush() throws Exception { + String sha = lfsPublication("missing entity".getBytes(StandardCharsets.UTF_8), false); + assertThatThrownBy(() -> copier.prepare(new Repository(1, "community", "source"), sha, "Fixture", "code", deadline())) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.code()).isEqualTo("FORK_COPY_FAILED")); + assertThat(objects.get("target")).isNull(); + assertClean(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void refusesLfsRedirectsAndActionsOutsideConfiguredGitea(boolean redirect) throws Exception { + String sha = lfsPublication("restricted entity".getBytes(StandardCharsets.UTF_8), true); + redirectLfs = redirect; + externalAction = !redirect; + assertThatThrownBy(() -> copier.prepare(new Repository(1, "community", "source"), sha, "Fixture", "code", deadline())) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.code()).isEqualTo("FORK_COPY_FAILED")); + assertThat(unexpectedRequests).hasValue(0); + assertClean(); + } + + @Test void waitsForGiteaMetadataBeforeReadingTheFirstBranchAndChecksExactCommit() { + String sha = "a".repeat(40); + when(gitea.inspect("target")).thenReturn( + new GiteaClient.RemoteRepository(2, "community", "target", true, false, true, "", "main"), + new GiteaClient.RemoteRepository(2, "community", "target", true, false, false, "", "main")); + when(gitea.head(any())).thenReturn(sha); + CliGitSnapshotCopier real = new CliGitSnapshotCopier(properties, community, commands, gitea, json); + real.verifyRemote(new Repository(2, "community", "target"), sha, deadline()); + verify(gitea, times(2)).inspect("target"); + verify(gitea).head(any()); + when(gitea.head(any())).thenReturn("b".repeat(40)); + assertThatThrownBy(() -> real.verifyRemote(new Repository(2, "community", "target"), sha, deadline())) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.code()).isEqualTo("FORK_COPY_FAILED")); + } + + @Test void refusesSuccessIfTargetLfsCannotBeDownloaded() throws Exception { + String sha = lfsPublication("target download must work".getBytes(StandardCharsets.UTF_8), true); + rejectTargetDownload = true; + ForkDeadline deadline = deadline(); + try (GitSnapshotCopier.PreparedSnapshot snapshot = copier.prepare(new Repository(1, "community", "source"), sha, + "Fixture", "code", deadline)) { + assertThatThrownBy(() -> snapshot.push(new Repository(2, "community", "target"))) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.code()).isEqualTo("FORK_COPY_FAILED")); + assertThat(deadline.writing()).isFalse(); + } + assertClean(); + } + + @Test void rejectsUnknownShaAndDiskQuotaAndCleansTemporaryData() throws Exception { + String sha = simplePublication(); + assertThatThrownBy(() -> copier.prepare(new Repository(1, "community", "source"), "b".repeat(40), "Fixture", "code", deadline())) + .isInstanceOf(BusinessException.class); + assertClean(); + properties.setMaxTemporaryBytes(1); + assertThatThrownBy(() -> copier.prepare(new Repository(1, "community", "source"), sha, "Fixture", "code", deadline())) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.code()).isEqualTo("FORK_RESOURCE_LIMIT")); + assertClean(); + } + + @Test void rejectsMissingBinaryAndAuthenticationFailure() throws Exception { + properties.setGitExecutable(root.resolve("missing-git").toString()); + assertThatThrownBy(copier::checkAvailable).isInstanceOfSatisfying(BusinessException.class, + error -> assertThat(error.code()).isEqualTo("FORK_UNAVAILABLE")); + properties.setGitExecutable("git"); + CliGitSnapshotCopier http = new CliGitSnapshotCopier(properties, community, commands, gitea, json); + assertThatThrownBy(() -> http.prepare(new Repository(1, "community", "source"), "c".repeat(40), "Fixture", "code", deadline())) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.code()).isEqualTo("FORK_COPY_FAILED")); + assertClean(); + } + + @Test void marksExplicitPushAuthenticationRejectionAsDefiniteFailure() throws Exception { + String sha = simplePublication(); + rejectGitPush = true; + ForkDeadline deadline = deadline(); + try (GitSnapshotCopier.PreparedSnapshot snapshot = copier.prepare(new Repository(1, "community", "source"), sha, + "Fixture", "code", deadline)) { + assertThatThrownBy(() -> snapshot.push(new Repository(2, "community", "target"))) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.code()).isEqualTo("FORK_COPY_FAILED")); + assertThat(deadline.writing()).isFalse(); + } + assertClean(); + } + + private void serveLfs(HttpExchange exchange) throws IOException { + try (HttpExchange request = exchange) { + String path = request.getRequestURI().getPath(); + String[] parts = path.split("/"); + if (parts.length < 3 || !Set.of("source", "target").contains(parts[1])) { + unexpectedRequests.incrementAndGet(); + request.sendResponseHeaders(401, -1); + return; + } + String repository = parts[1]; + Map stored = objects.computeIfAbsent(repository, ignored -> new ConcurrentHashMap<>()); + if (path.endsWith("/objects/batch")) { + if (redirectLfs) { + request.getResponseHeaders().set("Location", origin + "/forbidden"); + request.sendResponseHeaders(307, -1); + return; + } + JsonNode input = json.readTree(request.getRequestBody()); + ObjectNode response = json.createObjectNode().put("transfer", "basic"); + ArrayNode entries = response.putArray("objects"); + for (JsonNode object : input.path("objects")) { + String oid = object.path("oid").asText(); + ObjectNode entry = entries.addObject().put("oid", oid).put("size", object.path("size").asLong()); + boolean download = input.path("operation").asText().equals("download"); + if (download && (!stored.containsKey(oid) || repository.equals("target") && rejectTargetDownload)) { + entry.putObject("error").put("code", 404).put("message", "fixture missing object"); + } else if (download || !stored.containsKey(oid)) { + entry.putObject("actions").putObject(download ? "download" : "upload") + .put("href", externalAction ? origin.replace("127.0.0.1", "localhost") + "/forbidden" + : origin + "/" + repository + "/objects/" + oid); + } + } + byte[] body = json.writeValueAsBytes(response); + request.getResponseHeaders().set("Content-Type", "application/vnd.git-lfs+json"); + request.sendResponseHeaders(200, body.length); + request.getResponseBody().write(body); + } else if (path.contains("/objects/")) { + String oid = parts[parts.length - 1]; + if (request.getRequestMethod().equals("PUT")) { + stored.put(oid, request.getRequestBody().readAllBytes()); + request.sendResponseHeaders(200, -1); + } else if (stored.containsKey(oid)) { + byte[] bytes = stored.get(oid); + request.sendResponseHeaders(200, bytes.length); + request.getResponseBody().write(bytes); + } else request.sendResponseHeaders(404, -1); + } else request.sendResponseHeaders(404, -1); + } + } +} diff --git a/community-service/src/test/java/com/linkerbot/community/service/fork/ForkExecutorTest.java b/community-service/src/test/java/com/linkerbot/community/service/fork/ForkExecutorTest.java new file mode 100644 index 0000000..ea7f55d --- /dev/null +++ b/community-service/src/test/java/com/linkerbot/community/service/fork/ForkExecutorTest.java @@ -0,0 +1,84 @@ +package com.linkerbot.community.service.fork; + +import static org.assertj.core.api.Assertions.*; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.*; + +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.config.ForkProperties; +import java.time.Duration; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.MDC; + +class ForkExecutorTest { + @Test void unavailableRuntimeRejectsBeforeExecutingBusinessWork() { + GitSnapshotCopier copier = mock(GitSnapshotCopier.class); + doThrow(new BusinessException(503, "FORK_UNAVAILABLE", "unavailable")).when(copier).checkAvailable(); + ForkExecutor executor = new ForkExecutor(new ForkProperties(), copier); + AtomicBoolean called = new AtomicBoolean(); + try { + executor.inspectRuntime(); + assertThatThrownBy(() -> executor.execute(deadline -> { called.set(true); return 1; })) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.code()).isEqualTo("FORK_UNAVAILABLE")); + assertThat(called).isFalse(); + } finally { executor.close(); } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void timeoutRetainsCapacityUntilActualWorkAndCleanupEnd(boolean writing) throws Exception { + ForkProperties properties = new ForkProperties(); + properties.setTimeout(Duration.ofMillis(150)); + properties.setMaxConcurrent(1); + ForkExecutor executor = new ForkExecutor(properties, mock(GitSnapshotCopier.class)); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicBoolean cleaned = new AtomicBoolean(); + try (ExecutorService callers = Executors.newVirtualThreadPerTaskExecutor()) { + Future result = callers.submit(() -> { + try { + executor.execute(deadline -> { + deadline.projectCode("111111111111"); + deadline.onClose(() -> cleaned.set(true)); + if (writing) deadline.beginWrite(); + entered.countDown(); + while (release.getCount() != 0) { + try { release.await(); } + catch (InterruptedException error) { interrupted.countDown(); } + } + return 1; + }); + throw new AssertionError("timeout expected"); + } catch (BusinessException error) { return error; } + }); + assertThat(entered.await(2, TimeUnit.SECONDS)).isTrue(); + BusinessException error = result.get(2, TimeUnit.SECONDS); + assertThat(error.code()).isEqualTo(writing ? "REMOTE_RESULT_UNKNOWN" : "FORK_TIMEOUT"); + assertThat(error.details()).isEqualTo(java.util.Map.of("projectCode", "111111111111")); + assertThat(interrupted.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(cleaned).isFalse(); + assertThatThrownBy(() -> executor.execute(deadline -> 2)) + .isInstanceOfSatisfying(BusinessException.class, busy -> assertThat(busy.code()).isEqualTo("FORK_BUSY")); + release.countDown(); + await().atMost(Duration.ofSeconds(2)).untilAsserted(() -> assertThat(executor.execute(deadline -> 3)).isEqualTo(3)); + assertThat(cleaned).isTrue(); + } finally { release.countDown(); executor.close(); } + } + + @Test void propagatesRequestContextAndDoesNotBindDeadlineToCaller() { + ForkExecutor executor = new ForkExecutor(new ForkProperties(), mock(GitSnapshotCopier.class)); + MDC.put("requestId", "fixture-request"); + try { + assertThat(executor.execute(deadline -> { + assertThat(ForkDeadline.current()).isSameAs(deadline); + return MDC.get("requestId"); + })).isEqualTo("fixture-request"); + assertThat(ForkDeadline.current()).isNull(); + } finally { MDC.remove("requestId"); executor.close(); } + } +} diff --git a/community-service/src/test/java/com/linkerbot/community/service/fork/GitCommandRunnerTest.java b/community-service/src/test/java/com/linkerbot/community/service/fork/GitCommandRunnerTest.java new file mode 100644 index 0000000..a67e86e --- /dev/null +++ b/community-service/src/test/java/com/linkerbot/community/service/fork/GitCommandRunnerTest.java @@ -0,0 +1,62 @@ +package com.linkerbot.community.service.fork; + +import static org.assertj.core.api.Assertions.*; +import static org.awaitility.Awaitility.await; + +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.config.ForkProperties; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.Duration; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class GitCommandRunnerTest { + @TempDir Path root; + + private GitCommandRunner script(String content) throws Exception { + Path executable = root.resolve("fixture-git"); + Files.writeString(executable, "#!/bin/sh\n" + content); + Files.setPosixFilePermissions(executable, PosixFilePermissions.fromString("rwx------")); + ForkProperties properties = new ForkProperties(); + properties.setGitExecutable(executable.toString()); + return new GitCommandRunner(properties); + } + + @Test void timeoutTerminatesTheProcessAndItsChild() throws Exception { + Path pid = root.resolve("child.pid"); + GitCommandRunner runner = script("sleep 60 &\nchild=$!\nprintf '%s' \"$child\" > '" + pid + "'\nwait \"$child\"\n"); + ForkDeadline deadline = new ForkDeadline(Duration.ofSeconds(10)); + try (java.util.concurrent.ExecutorService worker = java.util.concurrent.Executors.newVirtualThreadPerTaskExecutor()) { + java.util.concurrent.Future task = worker.submit(() -> runner.run(null, deadline, "blocking", Map.of(), Map.of(), "--version")); + await().atMost(Duration.ofSeconds(5)).until(() -> Files.exists(pid) && Files.size(pid) > 0); + deadline.cancel("FORK_TIMEOUT"); + assertThatThrownBy(() -> task.get(3, java.util.concurrent.TimeUnit.SECONDS)) + .isInstanceOf(java.util.concurrent.ExecutionException.class) + .hasCauseInstanceOf(BusinessException.class) + .satisfies(error -> assertThat(((BusinessException) error.getCause()).code()).isEqualTo("FORK_TIMEOUT")); + } + long child = Long.parseLong(Files.readString(pid)); + await().atMost(Duration.ofSeconds(2)).untilAsserted(() -> + assertThat(ProcessHandle.of(child).map(ProcessHandle::isAlive).orElse(false)).isFalse()); + } + + @Test void limitsUntrustedProcessOutput() throws Exception { + GitCommandRunner runner = script("exec head -c 16000000 /dev/zero\n"); + assertThatThrownBy(() -> runner.run(null, new ForkDeadline(Duration.ofSeconds(5)), "verbose", Map.of(), Map.of(), "--version")) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.code()).isEqualTo("FORK_RESOURCE_LIMIT")); + } + + @Test void putsCredentialsInEnvironmentWithoutArgumentsOrInheritedGitConfiguration() throws Exception { + GitCommandRunner runner = script("printf '%s|%s|%s' \"$*\" \"$GIT_CONFIG_VALUE_6\" \"$GIT_CONFIG_GLOBAL\"\n"); + GitCommandRunner.Result result = runner.run(null, new ForkDeadline(Duration.ofSeconds(3)), "environment", + Map.of("http.http://fixture.invalid/.extraHeader", "Authorization: Basic fixture-only"), Map.of(), "ls-remote", "http://fixture.invalid/repo.git"); + assertThat(result.exitCode()).isZero(); + String[] parts = result.output().split("\\|", -1); + assertThat(parts[0]).doesNotContain("Authorization", "fixture-only"); + assertThat(parts[1]).isEqualTo("Authorization: Basic fixture-only"); + assertThat(parts[2]).isEqualTo("/dev/null"); + } +} diff --git a/community-service/src/test/java/com/linkerbot/community/service/fork/ProjectForkStateTest.java b/community-service/src/test/java/com/linkerbot/community/service/fork/ProjectForkStateTest.java new file mode 100644 index 0000000..cabac5a --- /dev/null +++ b/community-service/src/test/java/com/linkerbot/community/service/fork/ProjectForkStateTest.java @@ -0,0 +1,96 @@ +package com.linkerbot.community.service.fork; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import com.linkerbot.community.bean.Contracts.ForkInput; +import com.linkerbot.community.bean.entity.Project; +import com.linkerbot.community.bean.entity.User; +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.UserService; +import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.config.ForkProperties; +import com.linkerbot.community.service.gitea.GiteaClient; +import com.linkerbot.community.service.impl.ProjectServiceImpl; +import com.linkerbot.community.service.mapper.*; +import com.linkerbot.community.service.oss.UrdfResources; +import com.linkerbot.community.service.support.ProjectLocks; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.beans.factory.support.StaticListableBeanFactory; + +class ProjectForkStateTest { + @ParameterizedTest + @ValueSource(strings = {"initial", "association", "completion"}) + void databaseWriteFailuresRemainUncertainWithCode(String stage) { + boolean completion = stage.equals("completion"); + Project source = new Project(); + source.setId(1L); + source.setProjectCode("111111111111"); + source.setPublishedCommitSha("a".repeat(40)); + source.setProjectType("GENERAL"); + source.setName("来源"); + source.setDescription(""); + source.setGiteaRepoId(1L); + source.setGiteaOwner("community"); + source.setGiteaRepoName("p_111111111111"); + ProjectMapper projects = mock(ProjectMapper.class); + when(projects.selectOne(any())).thenReturn(source); + AtomicReference target = new AtomicReference<>(); + when(projects.insert(any(Project.class))).thenAnswer(call -> { + Project project = call.getArgument(0); + project.setId(2L); + target.set(project); + if (stage.equals("initial")) throw new IllegalStateException("simulated insertion receipt lost"); + return 1; + }); + AtomicReference failed = new AtomicReference<>(false); + when(projects.updateById(any(Project.class))).thenAnswer(call -> { + Project project = call.getArgument(0); + if (!failed.get() && (completion == "READY".equals(project.getRepoStatus()))) { + failed.set(true); + throw new IllegalStateException("simulated database write failure"); + } + return 1; + }); + User user = new User(); + user.setNickname("测试用户"); + user.setUserCode("user-code"); + UserService users = mock(UserService.class); + when(users.require(1)).thenReturn(user); + GiteaClient gitea = mock(GiteaClient.class); + when(gitea.createRepository(anyString(), eq(false))).thenReturn(new GiteaClient.Repository(2, "community", "target")); + GitSnapshotCopier copier = mock(GitSnapshotCopier.class); + GitSnapshotCopier.PreparedSnapshot snapshot = mock(GitSnapshotCopier.PreparedSnapshot.class); + when(copier.prepare(any(), anyString(), anyString(), anyString(), any())).thenReturn(snapshot); + when(snapshot.push(any())).thenReturn("b".repeat(40)); + ForkExecutor executor = new ForkExecutor(new ForkProperties(), copier); + StaticListableBeanFactory factory = new StaticListableBeanFactory(); + factory.addBean("forks", executor); + factory.addBean("snapshots", copier); + ProjectLocks locks = new ProjectLocks(); + ProjectServiceImpl service = new ProjectServiceImpl(projects, mock(LikeMapper.class), mock(CommentMapper.class), users, + gitea, new CommunityProperties(), locks, factory.getBeanProvider(UrdfResources.class), + factory.getBeanProvider(ForkExecutor.class), factory.getBeanProvider(GitSnapshotCopier.class)); + try { + assertThatThrownBy(() -> service.fork(1, source.getProjectCode(), new ForkInput(source.getPublishedCommitSha(), "副本"))) + .isInstanceOfSatisfying(BusinessException.class, error -> { + assertThat(error.code()).isEqualTo("PROJECT_INITIALIZATION_ERROR"); + assertThat(error.details()).isEqualTo(Map.of("projectCode", target.get().getProjectCode())); + }); + assertThat(target.get().getRepoStatus()).isEqualTo("CREATING"); + assertThat(target.get().getGiteaRepoId()).isEqualTo(stage.equals("initial") ? null : 2L); + assertThatThrownBy(() -> locks.write(target.get().getProjectCode(), () -> true)) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.code()).isEqualTo("REMOTE_RESULT_UNKNOWN")); + if (completion) verify(snapshot).push(any()); + else verify(snapshot, never()).push(any()); + if (stage.equals("initial")) { + verify(copier, never()).prepare(any(), anyString(), anyString(), anyString(), any()); + verify(snapshot, never()).close(); + } else verify(snapshot).close(); + } finally { executor.close(); } + } +} diff --git a/community-service/src/test/java/com/linkerbot/community/service/oss/AliyunOssObjectStoreTest.java b/community-service/src/test/java/com/linkerbot/community/service/oss/AliyunOssObjectStoreTest.java index ecda76a..b7264fb 100644 --- a/community-service/src/test/java/com/linkerbot/community/service/oss/AliyunOssObjectStoreTest.java +++ b/community-service/src/test/java/com/linkerbot/community/service/oss/AliyunOssObjectStoreTest.java @@ -4,6 +4,12 @@ import static org.assertj.core.api.Assertions.*; import com.linkerbot.community.common.BusinessException; import com.linkerbot.community.service.config.OssProperties; +import com.linkerbot.community.service.config.ForkProperties; +import com.linkerbot.community.service.fork.ForkExecutor; +import com.linkerbot.community.service.fork.GitSnapshotCopier; +import java.time.Duration; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.mock; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpServer; import java.io.IOException; @@ -16,6 +22,7 @@ import org.junit.jupiter.api.Test; /** 使用本机 HTTP 替身验证真实 SDK 的 V4 请求和 XML 解析,不访问阿里云。 */ class AliyunOssObjectStoreTest { + private volatile long responseDelayMs; private HttpServer server; private AliyunOssObjectStore store; private final AtomicReference copySource = new AtomicReference<>(); @@ -39,6 +46,10 @@ class AliyunOssObjectStoreTest { private void respond(HttpExchange exchange) throws IOException { try (HttpExchange request = exchange) { + if (responseDelayMs > 0) { + try { Thread.sleep(responseDelayMs); } + catch (InterruptedException error) { Thread.currentThread().interrupt(); } + } authorization.set(request.getRequestHeaders().getFirst("Authorization")); request.getResponseHeaders().set("x-oss-request-id", "fake-request-id"); if (request.getRequestMethod().equals("HEAD")) { @@ -68,6 +79,22 @@ class AliyunOssObjectStoreTest { if (server != null) server.stop(0); } + @Test + void forkBudgetBoundsSdkRequestsAndClosesItsDedicatedClient() { + store.head("vibemake_dev/urdf/warm/main.urdf"); + ForkProperties fork = new ForkProperties(); + fork.setTimeout(Duration.ofMillis(400)); + fork.setMaxConcurrent(1); + ForkExecutor executor = new ForkExecutor(fork, mock(GitSnapshotCopier.class)); + responseDelayMs = 2000; + try { + assertThatThrownBy(() -> executor.execute(deadline -> store.head("vibemake_dev/urdf/slow/main.urdf"))) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.code()).isEqualTo("FORK_TIMEOUT")); + await().atMost(Duration.ofSeconds(1)).untilAsserted(() -> + assertThat(executor.execute(deadline -> 1)).isEqualTo(1)); + } finally { executor.close(); } + } + @Test void realSdkReadsMetadataAndCopiesWithV4AuthenticationAndSourceEtag() { OssObjectStore.Metadata metadata = store.head("vibemake_dev/urdf/a/机械 臂.urdf"); diff --git a/docs/community-api.md b/docs/community-api.md index 8cea1c7..83659a2 100644 --- a/docs/community-api.md +++ b/docs/community-api.md @@ -40,7 +40,16 @@ | 接口调试 | `/swagger-ui.html` | | OpenAPI | `/v3/api-docs` | -当前后端没有配置跨域白名单。开发时通过前端开发服务器代理 `/api`,部署时通过同域反向代理访问;仅允许 OPTIONS 请求不代表已经支持浏览器跨域访问。 +后端按环境为 `/api/**` 配置跨域白名单,配置项为 `community.cors.allowed-origins`: + +| 后端环境 | 允许的前端 Origin | +| --- | --- | +| test(默认) | `https://dev-vibemake.robotquan.com` | +| prod | `https://vibemake.robotquan.com` | + +允许 `GET、HEAD、POST、PUT、PATCH、DELETE、OPTIONS`,请求头允许 `Authorization、Content-Type、Accept`,预检缓存 3600 秒。前端继续通过 `Authorization: Bearer <平台 token>` 鉴权,无需设置 `credentials: 'include'`;后端不启用跨域 Cookie 凭据。成功及鉴权、业务错误响应均支持上述来源,并暴露 `X-Request-Id`、`Content-Disposition` 响应头。 + +其他域名、HTTP 来源、非默认端口和另一环境的前端来源均不在默认白名单内。本地开发继续通过前端开发服务器代理 `/api`;部署代理需透传 `Origin`、预检请求及后端 CORS 响应头,避免重复添加跨域头。OSS 直传的跨域规则仍需单独在 Bucket 配置。 ### 统一响应 @@ -565,7 +574,7 @@ OSS 控制台配置步骤: 1. 准备指定地域的 Bucket,确认 test/prod 的根目录分别为 `vibemake_dev/`、`vibemake/`。目录是对象前缀,不是 Bucket 名。 2. 给服务端 RAM 用户授予对应环境目录的 `oss:PutObject` 权限,例如资源 `acs:oss:*:*:YOUR_BUCKET/vibemake_dev/project-covers/*`;正式环境改为 `vibemake/project-covers/*`,两个环境可使用各自凭据。 -3. 配置 Bucket CORS:AllowedOrigins 填真实前端开发、部署 Origin(含协议和端口),AllowedMethods 包含 POST,若使用 fetch 读图则增加 GET、HEAD;AllowedHeaders 允许 Content-Type,可按需要暴露 ETag、x-oss-request-id。调用签名接口仍走原有 `/api` 代理,不能用后端 OPTIONS 放行替代 OSS CORS。 +3. 配置 Bucket CORS:AllowedOrigins 填真实前端开发、部署 Origin(含协议和端口),AllowedMethods 包含 POST,若使用 fetch 读图则增加 GET、HEAD;AllowedHeaders 允许 Content-Type,可按需要暴露 ETag、x-oss-request-id。调用签名接口可按上述 API 跨域约定直接访问后端,或通过 `/api` 代理访问;后端 API 的 CORS 不能替代 OSS CORS。 4. 对封面目录配置公开 GetObject 读取权限,或使用能够公开返回图片的 CDN;私有源站通过 CDN 时需配置回源鉴权。对象沿用 Bucket 的访问规则,应用不修改整个 Bucket 的 ACL,也不生成临时下载链接。 5. 测试环境完成“获取签名 → 浏览器直传 → 保存项目 → 匿名读取图片”,核对对象 key 和浏览器网络请求;生产环境重复验收 `vibemake/` 路径。 @@ -922,9 +931,24 @@ commitSha 必须为当前 main 的完整 SHA,不能发布历史提交。成功 成功 data 为当前用户的新 Project,状态 READY、UNPUBLISHED;使用返回的新 projectCode 跳转工作区。类型和描述继承源项目,sourceProjectCode、sourceCommitSha 记录来源。 -Fork 复制发布版本的所有普通文件,不复制源仓库提交历史,不自动同步上游。允许同一用户多次 Fork。所有文件复制完成才返回 READY;不支持符号链接或子模块。 +Fork 使用 Git 仓库级快照复制:冻结校验通过的 publishedCommitSha,保留该提交完整的文件树,在新私有仓库 main 上创建一个无父提交的初始版本,不继承源仓库的历史、其他分支或标签,不自动同步上游。允许同一用户及多个用户多次 Fork 同一来源。新副本的提交 SHA 与 sourceCommitSha 不同,应通过工作区 tree 接口获取新版本。 -源项目发布版本已变化时返回 409 CONFLICT,刷新社区详情后由用户重新确认。Fork 是同步请求,可能耗时较长;提交期间显示加载状态并防止重复点击。超时、网络断开或失败后先查询工作区,不要自动重发。 +复制保留普通文件、执行权限、符号链接、`.gitmodules` 和子模块引用;不递归下载子模块依赖。Git LFS 指针及发布快照引用的实体会完整复制,并从副本独立下载校验。缺失的 LFS 实体会导致 Fork 失败。复制不使用普通文件接口的 10 MiB 读取上限,在线文件浏览、下载及 LFS 展开能力仍受原限制。URDF 按受理时绑定的当前清单独立复制到新目录,复制时校验 ETag,封面继承原 URL;Git、LFS 和 OSS 全部完成后才返回 READY。 + +后端部署需要 Git 和 git-lfs。默认 Fork 总预算5分钟、单实例最多同时2次,每次临时磁盘上限5 GiB(含 Git 对象、LFS 源缓存和副本校验缓存)。配置分别为 FORK_TIMEOUT、FORK_MAX_CONCURRENT、FORK_MAX_TEMPORARY_BYTES;临时目录可通过 FORK_TEMPORARY_DIRECTORY 指定。依赖不可用或并发超限时,在创建项目记录前拒绝。普通创建和其他 API 不依赖 Fork 运行环境。 + +| 错误码 | 行为 | +| --- | --- | +| FORK_UNAVAILABLE(503) | Git、git-lfs、临时目录或 Gitea 配置不可用,联系后端处理 | +| FORK_BUSY(429) | 达到 Fork 并发上限,稍后手动重试 | +| FORK_RESOURCE_LIMIT(413) | 临时磁盘或元数据超过限制,核对项目状态及服务端配额 | +| FORK_COPY_FAILED(502) | 快照、认证或 LFS 复制/校验失败,使用 requestId 排查 | +| FORK_TIMEOUT(504)/ FORK_CANCELLED(499) | 已超时或取消,先核对工作区再决定下一步 | +| REMOTE_RESULT_UNKNOWN(504) | 远端写入结果未确认,保留 CREATING,禁止自动重试 | + +创建记录后发生异常时,错误 data.projectCode 指向本次新副本。明确失败标记 FAILED;远端写入结果不明或回写异常保留 CREATING 并阻止后续写入。后端终止相关子进程、清理本地临时目录,不自动删除远端仓库或 OSS 对象,也不自动修复历史失败副本。 + +源项目发布版本已变化时返回 409 CONFLICT,刷新社区详情后由用户重新确认。Fork 是同步请求,可能耗时较长;前端和代理读取超时默认至少400秒,提交期间显示加载状态并防止重复点击。超时、网络断开或失败后先查询工作区,不要自动重发。 ## 评论和点赞 @@ -1078,9 +1102,10 @@ data:{"code":"OK","message":"成功","data":{"status":"NO_MATCH","reply":"暂未 | AI_INVALID_RESPONSE | 模型结果的操作、参数、编码或引用未通过校验,不执行该操作 | | AI_SEARCH_FAILURE | 服务内部异常,携带 requestId 联系后端;涉及写入时先核对工作区 | | NOT_FOUND / CONFLICT | Fork 来源不可访问或发布版本变化,不自动重新选择或换版本执行 | +| FORK_UNAVAILABLE / FORK_BUSY / FORK_RESOURCE_LIMIT / FORK_COPY_FAILED / FORK_TIMEOUT / FORK_CANCELLED | 按上文 Fork 错误码处理;error 事件不会改变已建立 SSE 的 HTTP 200 状态 | | GITEA_ERROR / REMOTE_RESULT_UNKNOWN / PROJECT_INITIALIZATION_ERROR | 项目初始化失败或结果待核对,优先使用 data.projectCode 查询工作区详情 | -默认每次模型调用30秒,整次处理65秒(包括仓库操作),全局并发4个、每用户滑动一分钟最多10次,超限不排队。主动取消通过 AbortController;服务端在检测连接断开后尽力取消上游,实际工作结束后释放并发名额。写入前检测取消,写入开始后超时或断开不保证回滚;即使没有收到项目编码,也应先查询工作区列表核对,不自动重发。不要在页面上同时启动多个 AI 请求;代理应关闭 SSE 缓冲,读取超时至少90秒。 +默认每次模型调用30秒,模型/搜索阶段总预算65秒;进入 forking 后切换为独立 Fork 预算(默认5分钟,另留15秒入口及终态余量),不再受搜索阶段65秒截止时间影响。SSE 容器超时覆盖搜索与 Fork 两阶段,心跳持续发送。全局 AI 并发4个、每用户滑动一分钟最多10次,Fork 另受单实例2次并发限制,超限不排队。主动取消通过 AbortController;服务端在检测连接断开后尽力取消上游,实际工作结束后释放并发名额。写入前检测取消,写入开始后超时或断开不保证回滚;即使没有收到项目编码,也应先查询工作区列表核对,不自动重发。不要在页面上同时启动多个 AI 请求;代理应关闭 SSE 缓冲,读取超时至少400秒。 ### fetch 接收 SSE 示例 @@ -1247,7 +1272,7 @@ const result = await chatAi( ## 前端请求示例 -以下示例使用浏览器 fetch 和 TypeScript,类型声明见前面的响应结构。API_BASE 使用 `/api`,配合同域反向代理或开发代理。 +以下示例使用浏览器 fetch 和 TypeScript,类型声明见前面的响应结构。API_BASE 默认使用 `/api`,配合同域反向代理或开发代理;从白名单前端直接跨域访问时,改为实际后端的 HTTPS 地址加 `/api`,继续传递 Bearer Token,无需设置 `credentials: 'include'`。 ### JSON 与 FormData 请求封装 @@ -1404,6 +1429,8 @@ function parseUtcTime(value: UtcDateTime): Date { 使用 Vite 时可在现有配置中加入以下 server 配置,保留 `/api` 路径,不进行 rewrite: +本地代理的 `changeOrigin` 会修改上游 Host,但可能保留浏览器的 localhost Origin。为避免本地写请求被后端白名单拒绝,示例通过 [Vite 的 configure 回调](https://vite.dev/config/server-options#server-proxy)在本地代理转发时移除 Origin,Authorization 保持原样;线上部署代理仍需透传真实 Origin。 + ```typescript export default { server: { @@ -1411,6 +1438,11 @@ export default { '/api': { target: 'http://localhost:8993', changeOrigin: true, + configure(proxy) { + proxy.on('proxyReq', (proxyReq) => { + proxyReq.removeHeader('origin'); + }); + }, }, }, }, diff --git a/scripts/test-gitea.py b/scripts/test-gitea.py index facf48c..dc24730 100644 --- a/scripts/test-gitea.py +++ b/scripts/test-gitea.py @@ -19,6 +19,9 @@ if config.exists(): if not environment.get("GITEA_TOKEN"): raise SystemExit("请先配置 GITEA_TOKEN 和 GITEA_ORGANIZATION") environment["COMMUNITY_LIVE_GITEA"] = "true" +# 本脚本只验证仓库协议;OSS/AI 有独立测试,避免本机其他服务配置阻止启动。 +environment["AI_ENABLED"] = "false" +environment["OSS_ENABLED"] = "false" raise SystemExit(subprocess.call([ "mvn", "-q", "-Dtest=GiteaLiveTest", "-Dsurefire.failIfNoSpecifiedTests=false", "test" ], cwd=root, env=environment))