From 674f0400db067d7ea51598eec7b7b3c0c677e3b1 Mon Sep 17 00:00:00 2001 From: lxp <2770281812@qq.com> Date: Mon, 2 Nov 2026 16:53:17 +0800 Subject: [PATCH] =?UTF-8?q?=E9=A1=B9=E7=9B=AE=E4=B8=8B=E8=BD=BDZIP?= =?UTF-8?q?=E3=80=81TAR.GZ?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 21 +- docs/vibemake-api.md | 97 ++++++++- .../vibemake/api/ArchiveApiTest.java | 206 ++++++++++++++++++ .../vibemake/api/FakeGiteaClient.java | 18 ++ .../linkerbot/vibemake/api/GiteaLiveTest.java | 50 +++++ .../vibemake/testing/ArchiveFixtures.java | 94 ++++++++ .../service/config/LimitProperties.java | 1 + .../vibemake/service/gitea/GiteaClient.java | 15 ++ .../service/gitea/RestGiteaClient.java | 60 +++++ .../service/state/HttpSafetyFilter.java | 2 + .../service/archive/ArchiveServiceTest.java | 137 ++++++++++++ .../service/gitea/GiteaArchiveTest.java | 204 +++++++++++++++++ .../vibemake/service/state/SafetyTest.java | 21 ++ 13 files changed, 923 insertions(+), 3 deletions(-) create mode 100644 vibemake-api/src/test/java/com/linkerbot/vibemake/api/ArchiveApiTest.java create mode 100644 vibemake-common/src/test/java/com/linkerbot/vibemake/testing/ArchiveFixtures.java create mode 100644 vibemake-service/src/test/java/com/linkerbot/vibemake/service/archive/ArchiveServiceTest.java create mode 100644 vibemake-service/src/test/java/com/linkerbot/vibemake/service/gitea/GiteaArchiveTest.java diff --git a/README.md b/README.md index 51eb7dc..b9fdd72 100644 --- a/README.md +++ b/README.md @@ -79,13 +79,30 @@ SPRING_PROFILES_ACTIVE=prod java -Djava.awt.headless=true -jar vibemake-api/targ 业务数据库与接口中的日期时间统一使用 `Asia/Shanghai`,不依赖服务器默认时区;接口中不带时区后缀的时间需按 `+08:00` 解析。OSS 签名协议字段保留 UTC,Git 提交日期保留原时区。已有 UTC 数据库升级前,先按[业务时间迁移说明](docs/admin-database.md#业务时间由-utc-迁移到上海时间)完成一次性转换,再同时部署社区端、管理端与导入工具;前端同步调整解析方式。 +## 仓库归档下载配置 + +用户端工作区与社区均支持 ZIP、TAR.GZ 仓库归档:工作区需登录且只能下载本人项目,支持当前 main 或指定历史 SHA;社区可匿名下载当前已发布 SHA。后端鉴权后直接流式转发 Gitea 原生归档,不向浏览器返回 Gitea 凭据或内部地址,无需数据库变更,也不新增本地 Git 或临时磁盘依赖。 + +| 环境变量 | 默认值 | 说明 | +| --- | --- | --- | +| ARCHIVE_MAX_BYTES | 1073741824 | 压缩包最大字节数,默认 1 GiB,独立于单文件与 multipart 限制 | +| ARCHIVE_TIMEOUT | 5m | 获取当前 SHA、归档响应及响应体读取的上游预算 | +| ARCHIVE_MAX_CONCURRENT | 2 | 每实例独立归档并发,超限返回 ARCHIVE_BUSY,不占用普通文件并发额度 | +| RATE_ARCHIVE_READ_PER_MINUTE | 10 | 每 IP 每分钟归档额度,工作区与社区共用 | + +对应配置为 `vibemake.archive.max-bytes`、`timeout`、`max-concurrent` 和 `vibemake.limits.per-minute.archive-read`;大小必须大于 0,并发为 1~32,超时为 100 毫秒~1 小时。沿用 Gitea 服务端 Token,需具有目标私有仓库的读取权限。上线前确认 `/api/v1/repos/{owner}/{repo}/archive/{SHA}.zip`、`{SHA}.tar.gz` 均直接返回 HTTP 200 文件;本服务不跟随重定向。Gitea 使用对象存储归档并返回重定向时,应配置 Gitea 直接输出归档(1.26.4 的 `[repository] STREAM_ARCHIVES=true` 支持此行为)。 + +代理读取超时至少 400 秒;调大 ARCHIVE_TIMEOUT 时同步增加代理超时,流式响应应关闭代理缓冲和缓存。中途超限、超时或连接断开会停止传输,已发送附件后不能再返回 JSON,下载文件可能不完整,应丢弃并手动重新下载。本期不支持断点续传、下载任务或进度查询。 + +归档沿用 Gitea 原生规则,LFS 保留指针,不下载实体,不递归下载子模块,也不包含单独绑定于 OSS 的封面、URDF、STEP、GLB、零件、拼装及行走资源。接口及支持取消的 fetch 示例见[仓库 ZIP、TAR.GZ 下载](docs/vibemake-api.md#仓库-ziptargz-下载)。 + ## 验证 ```bash mvn clean verify ``` -普通测试固定使用 test 配置,覆盖连接为临时 H2 MySQL 模式及模拟 Gitea/AI/OSS,验证登录、权限、提交冲突、发布快照、Fork、评论、点赞、URDF 直传核验和独立复制,以及 AI 筛选、引用校验、创建、Fork 与候选选择、SSE、限流、超时和取消,不访问真实数据库、仓库、对象存储或模型。未加载外部服务凭据时,执行 `AI_ENABLED=false OSS_ENABLED=false mvn test`;AI 流程测试自行启用模拟模型。完整安装验证可执行 `AI_ENABLED=false OSS_ENABLED=false mvn clean install`。 +普通测试固定使用 test 配置,覆盖连接为临时 H2 MySQL 模式及模拟 Gitea/AI/OSS,验证登录、权限、提交冲突、发布快照、Fork、评论、点赞、URDF 直传核验和独立复制,以及 AI 筛选、引用校验、创建、Fork 与候选选择、SSE、限流、超时和取消,不访问真实数据库、仓库、对象存储或模型。未加载外部服务凭据时,执行 `AI_ENABLED=false VIBEMAKE_AI_ENABLED=false OSS_ENABLED=false mvn verify`;其中 `VIBEMAKE_AI_ENABLED` 用于覆盖管理端当前固定开启的 AI 配置,AI 流程测试自行启用模拟模型。完整安装验证可执行 `AI_ENABLED=false VIBEMAKE_AI_ENABLED=false OSS_ENABLED=false mvn clean install`。 真实 Gitea 联调: @@ -93,7 +110,7 @@ mvn clean verify python3 scripts/test-gitea.py ``` -该测试使用临时 H2 数据库,在配置的组织创建临时项目仓库,验证私有仓库、多文件提交、中文路径、二进制上传、固定发布版本及快照 Fork,结束后仅删除本次测试创建的仓库。Gitea 26.4.0 内置的 1.26.4 支持批量文件变更;以服务器 /swagger.v1.json 为对接依据。 +该测试使用临时 H2 数据库,在配置的组织创建临时项目仓库,验证私有仓库、多文件提交、中文路径、二进制上传、ZIP/TAR.GZ 归档、固定发布版本及快照 Fork,结束后仅删除本次测试创建的仓库。归档联调核对中文路径、二进制内容、LFS 指针及发布后继续提交时的版本隔离。Gitea 26.4.0 内置的 1.26.4 支持批量文件变更;以服务器 /swagger.v1.json 为对接依据。 ## 测试 Gitea 组织仓库复制到生产 diff --git a/docs/vibemake-api.md b/docs/vibemake-api.md index f55d390..e63f377 100644 --- a/docs/vibemake-api.md +++ b/docs/vibemake-api.md @@ -55,7 +55,7 @@ ### 统一响应 -除成功的文件下载和 AI 对话 SSE 外,响应为以下 JSON 结构;SSE 每个事件的 data 也使用此包装。所有成功接口使用 HTTP 200,创建、删除、退出登录也不返回 201 或 204。 +除成功的文件或仓库归档下载和 AI 对话 SSE 外,响应为以下 JSON 结构;SSE 每个事件的 data 也使用此包装。所有成功接口使用 HTTP 200,创建、删除、退出登录也不返回 201 或 204。 ```json { @@ -127,6 +127,7 @@ Git SHA 使用完整的 40 或 64 位十六进制字符串,不使用缩写。 | DELETE | `/api/workspace/projects/{projectCode}/urdf` | 是 | null | | GET | `/api/workspace/projects/{projectCode}/tree` | 是 | Tree | | GET | `/api/workspace/projects/{projectCode}/files` | 是 | ProjectFile 或下载二进制 | +| GET | `/api/workspace/projects/{projectCode}/archive` | 是 | 当前或指定版本的 ZIP、TAR.GZ 仓库归档 | | POST | `/api/workspace/projects/{projectCode}/commits` | 是 | Commit | | GET | `/api/workspace/projects/{projectCode}/commits` | 是 | PageResult<HistoryEntry> | | POST | `/api/workspace/projects/{projectCode}/uploads/commit` | 是 | Commit | @@ -136,6 +137,7 @@ Git SHA 使用完整的 40 或 64 位十六进制字符串,不使用缩写。 | GET | `/api/community/projects/{projectCode}` | 可选 | Project | | GET | `/api/community/projects/{projectCode}/tree` | 可选 | Tree | | GET | `/api/community/projects/{projectCode}/files` | 可选 | ProjectFile 或下载二进制 | +| GET | `/api/community/projects/{projectCode}/archive` | 可选 | 当前已发布版本的 ZIP、TAR.GZ 仓库归档 | | POST | `/api/community/projects/{projectCode}/forks` | 是 | Project | | GET | `/api/community/projects/{projectCode}/comments` | 可选 | PageResult<Comment> | | POST | `/api/community/projects/{projectCode}/comments` | 是 | Comment | @@ -826,6 +828,99 @@ entries 仅包含当前目录的直接子项,目录在前,名称升序。展 禁止绝对路径、反斜杠、空路径段、`.`、`..`、`.git` 路径段、冒号及控制字符。读取普通文件接口不支持符号链接或 Git 子模块。使用 URLSearchParams 处理查询编码,不自行拼接未编码的 path。 +## 仓库 ZIP、TAR.GZ 下载 + +- `GET /api/workspace/projects/{projectCode}/archive`:需要登录,只允许下载本人的未删除项目,仓库必须 READY。 +- `GET /api/community/projects/{projectCode}/archive`:可匿名访问,只允许下载 READY、已发布且未删除项目的当前发布版本。 + +| 参数 | 工作区 | 社区 | 说明 | +| --- | --- | --- | --- | +| format | 可选 | 可选 | 省略为 `zip`;仅接受 `zip`、`tar.gz`,区分大小写,空字符串或其他值返回 400 | +| commitSha | 可选 | 禁止 | 工作区省略时下载当前 main,也可传入本项目的完整历史 SHA;社区传入该参数即返回 400,包括空字符串 | + +示例:`GET /api/community/projects/{projectCode}/archive?format=tar.gz`。 + +服务端在请求受理时冻结完整 SHA,再向 Gitea 获取归档。发布 A 后工作区提交 B,社区仍下载 A;工作区默认下载 B,指定 A 的 SHA 可下载 A。下载过程中 main 或发布版本变化不会切换本次版本;撤回或删除后,新发起的社区下载返回 404。 + +成功返回 HTTP 200 二进制附件,不使用 ApiResponse 包装。ZIP 为 `application/zip`,TAR.GZ 为 `application/gzip`,文件名为 `{projectCode}-{SHA前12位}.{format}`;带 `Content-Disposition: attachment`、`Cache-Control: no-store` 和 `X-Request-Id`。归档内的根目录、文件层级、符号链接和 `.gitattributes` 导出规则沿用 Gitea 原生行为,不包含 Git 提交历史。 + +**这是 Git 仓库归档,不是项目全部资源打包。** LFS 文件保留指针,不替换为真实大文件;子模块依赖不递归下载。单独绑定于 OSS 的封面、URDF、STEP、GLB、零件、拼装及行走资源不会加入归档;如果仓库本身包含同类文件,则按仓库内容正常导出。 + +归档使用独立配置,不受普通单文件读取 50 MiB、仓库上传 200 MiB 限制影响: + +| 配置 | 默认值 | 含义 | +| --- | --- | --- | +| ARCHIVE_MAX_BYTES | 1073741824(1 GiB) | 本次下载的压缩包字节数上限 | +| ARCHIVE_TIMEOUT | 5m | 获取当前版本、等待归档及读取上游响应体的预算 | +| ARCHIVE_MAX_CONCURRENT | 2 | 单实例同时处理的归档下载数,独立于普通文件并发,超限不排队 | +| RATE_ARCHIVE_READ_PER_MINUTE | 10 | 每 IP 每分钟下载请求数,工作区和社区共用额度 | + +| HTTP / code | 处理 | +| --- | --- | +| 400 BAD_REQUEST | 检查格式和 SHA;社区不要提交 commitSha | +| 401 UNAUTHORIZED | 工作区下载先登录;已失效 Token 不应携带到社区请求 | +| 404 NOT_FOUND | 项目不存在、无权访问、未公开、已撤回、已删除,或提交不存在 | +| 409 CONFLICT | 工作区仓库尚未就绪 | +| 413 ARCHIVE_TOO_LARGE | 压缩包超过归档大小上限 | +| 429 ARCHIVE_BUSY / RATE_LIMITED | 并发或频率超限,按 Retry-After 稍后手动重试 | +| 502 GITEA_ERROR | 上游异常、重定向或未返回有效归档,使用 requestId 排查 | +| 503 GITEA_NOT_CONFIGURED | 服务端仓库凭据尚未配置 | +| 504 ARCHIVE_TIMEOUT | 等待或读取超时,可稍后手动重新下载 | +| 499 ARCHIVE_CANCELLED | 客户端连接在发包前取消;主动停止时通常表现为 AbortError | + +错误发生在发送附件之前时返回上述 JSON 错误;开始传输后无法改写 HTTP 状态,服务端会停止传输,不向压缩包尾部追加 JSON。此时可能表现为网络中断或下载到不完整的压缩包,应丢弃残缺文件并手动重新下载。本期不提供断点续传、下载任务或进度查询。 + +工作区必须使用带 Bearer Token 的 fetch;不要将 Token 拼入 URL,也不要使用无法附加请求头的直接链接。成功后读取 Blob,失败先解析 JSON。以下示例支持两种入口和 AbortController,复用本文的 API_BASE、ApiError: + +```typescript +async function downloadRepositoryArchive( + scope: 'workspace' | 'community', + projectCode: string, + format: 'zip' | 'tar.gz' = 'zip', + options: { token?: string; commitSha?: string; signal?: AbortSignal } = {}, +) { + if (scope === 'workspace' && !options.token) throw new Error('请先登录'); + if (scope === 'community' && options.commitSha !== undefined) { + throw new Error('社区归档仅支持当前已发布版本'); + } + const query = new URLSearchParams({ format }); + if (options.commitSha !== undefined) query.set('commitSha', options.commitSha); + const response = await fetch( + `${API_BASE}/${scope}/projects/${encodeURIComponent(projectCode)}/archive?${query}`, + { + headers: options.token ? { Authorization: `Bearer ${options.token}` } : {}, + signal: options.signal, + }, + ); + if (!response.ok) { + const failure = await response.json(); + throw new ApiError(response.status, failure.code, failure.message, + failure.requestId ?? response.headers.get('X-Request-Id'), failure.data); + } + const disposition = response.headers.get('Content-Disposition') ?? ''; + const encodedName = disposition.match(/filename\*=UTF-8''([^;]+)/i)?.[1]; + const filename = encodedName ? decodeURIComponent(encodedName) : `${projectCode}.${format}`; + const blob = await response.blob(); // 完整接收后再触发保存。 + const objectUrl = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = objectUrl; + link.download = filename; + document.body.appendChild(link); + link.click(); + link.remove(); + setTimeout(() => URL.revokeObjectURL(objectUrl), 0); +} + +const downloadController = new AbortController(); +// 社区可省略 token;工作区传入平台 token。 +await downloadRepositoryArchive('workspace', projectCode, 'zip', { + token, signal: downloadController.signal, +}); +// 用户点击停止或离开页面时调用 downloadController.abort()。 +``` + +Blob 下载会占用浏览器存储或内存,不代表服务端将整个归档缓存在内存中;前端应限制重复点击,捕获 AbortError 和网络错误。 + ## 文件编辑与上传提交 ### 多文件提交 diff --git a/vibemake-api/src/test/java/com/linkerbot/vibemake/api/ArchiveApiTest.java b/vibemake-api/src/test/java/com/linkerbot/vibemake/api/ArchiveApiTest.java new file mode 100644 index 0000000..cf2d4ae --- /dev/null +++ b/vibemake-api/src/test/java/com/linkerbot/vibemake/api/ArchiveApiTest.java @@ -0,0 +1,206 @@ +package com.linkerbot.vibemake.api; + +import static org.assertj.core.api.Assertions.*; +import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkerbot.vibemake.service.gitea.GiteaClient; +import com.linkerbot.vibemake.service.gitea.GiteaFailure; +import com.linkerbot.vibemake.testing.ArchiveFixtures; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.autoconfigure.web.servlet.MockMvcPrint; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.http.MediaType; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:archives;MODE=MySQL;DB_CLOSE_DELAY=-1", + "spring.datasource.username=sa", "spring.datasource.password=", + "spring.datasource.driver-class-name=org.h2.Driver", + "vibemake.ai.enabled=false", "vibemake.oss.enabled=false", + "vibemake.limits.enabled=false", "vibemake.repository-write.worker-enabled=false", + "logging.level.root=WARN"}) +@ActiveProfiles("test") +@AutoConfigureMockMvc(print = MockMvcPrint.NONE) +@Import(WorkflowTest.Config.class) +class ArchiveApiTest { + @Autowired MockMvc mvc; + @Autowired ObjectMapper json; + @Autowired JdbcTemplate jdbc; + @MockitoSpyBean FakeGiteaClient gitea; + + @BeforeEach + void resetState() throws Exception { + reset(gitea); + gitea.reset(); + SchemaSupport.reset(jdbc); + } + + private JsonNode call(MockHttpServletRequestBuilder request) throws Exception { + return json.readTree(mvc.perform(request).andExpect(status().isOk()).andReturn() + .getResponse().getContentAsByteArray()).path("data"); + } + + private MockHttpServletRequestBuilder body(MockHttpServletRequestBuilder request, Object input) throws Exception { + return request.contentType(MediaType.APPLICATION_JSON).content(json.writeValueAsBytes(input)); + } + + private MockHttpServletRequestBuilder auth(MockHttpServletRequestBuilder request, String token) { + return request.header("Authorization", "Bearer " + token); + } + + private String login(String email) throws Exception { + String captcha = call(get("/api/auth/captcha")).path("captchaCode").asText(); + return call(body(post("/api/auth/login"), Map.of("email", email, + "captchaCode", captcha, "captchaAnswer", "ABCDE"))).path("token").asText(); + } + + private String create(String token) throws Exception { + return call(body(auth(post("/api/workspace/projects"), token), + Map.of("name", "归档测试", "projectType", "GENERAL"))).path("projectCode").asText(); + } + + private String head(String token, String code) throws Exception { + return call(auth(get("/api/workspace/projects/" + code + "/tree"), token)).path("commitSha").asText(); + } + + private String update(String token, String code, String sha, String content) throws Exception { + return call(body(auth(post("/api/workspace/projects/" + code + "/commits"), token), + Map.of("expectedCommitSha", sha, "message", "版本更新", "changes", java.util.List.of( + Map.of("action", "UPDATE_FILE", "path", "README.md", "content", content))))) + .path("commitSha").asText(); + } + + @ParameterizedTest + @ValueSource(strings = {"zip", "tar.gz"}) + void archivesSeparateWorkspaceAndPublishedVersions(String format) throws Exception { + String token = login("archive-versions@example.invalid"); + String code = create(token); + String first = head(token, code); + call(body(auth(put("/api/workspace/projects/" + code + "/publication"), token), Map.of("commitSha", first))); + String second = update(token, code, first, "版本 B"); + for (String context : java.util.List.of("community", "workspace", "history")) { + boolean workspace = !context.equals("community"); + String expected = context.equals("workspace") ? second : first; + MockHttpServletRequestBuilder request = get("/api/" + (workspace ? "workspace" : "community") + + "/projects/" + code + "/archive").param("format", format) + .header("Origin", "https://dev-vibemake.robotquan.com"); + if (workspace) auth(request, token); + if (context.equals("history")) request.param("commitSha", first); + MvcResult result = mvc.perform(request).andExpect(status().isOk()) + .andExpect(content().contentType(format.equals("zip") ? "application/zip" : "application/gzip")) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(header().string("Content-Disposition", containsString(code + "-" + expected.substring(0, 12)))) + .andExpect(header().exists("X-Request-Id")) + .andExpect(header().string("Access-Control-Expose-Headers", containsString("Content-Disposition"))) + .andReturn(); + Map files = ArchiveFixtures.read(format, result.getResponse().getContentAsByteArray()); + assertThat(new String(files.get("README.md"), StandardCharsets.UTF_8)) + .isEqualTo(context.equals("workspace") ? "版本 B" : "initial"); + } + } + + @Test + void permissionsVisibilityAndReadinessFollowExistingRules() throws Exception { + String token = login("archive-owner@example.invalid"); + String other = login("archive-other@example.invalid"); + String code = create(token); + String workspace = "/api/workspace/projects/" + code + "/archive"; + String community = "/api/community/projects/" + code + "/archive"; + mvc.perform(get(workspace)).andExpect(status().isUnauthorized()); + mvc.perform(auth(get(workspace), other)).andExpect(status().isNotFound()); + mvc.perform(get(community)).andExpect(status().isNotFound()); + // 社区管理员登录后也没有访问他人工作区的特权。 + jdbc.update("UPDATE sys_user SET role='ADMIN' WHERE email=?", "archive-other@example.invalid"); + mvc.perform(auth(get(workspace), other)).andExpect(status().isNotFound()); + String sha = head(token, code); + call(body(auth(put("/api/workspace/projects/" + code + "/publication"), token), Map.of("commitSha", sha))); + mvc.perform(get(community)).andExpect(status().isOk()).andExpect(content().contentType("application/zip")); + jdbc.update("UPDATE project SET repo_status='CREATING' WHERE project_code=?", code); + mvc.perform(auth(get(workspace), token)).andExpect(status().isConflict()); + mvc.perform(get(community)).andExpect(status().isNotFound()); + jdbc.update("UPDATE project SET repo_status='READY' WHERE project_code=?", code); + call(auth(delete("/api/workspace/projects/" + code + "/publication"), token)); + mvc.perform(get(community)).andExpect(status().isNotFound()); + call(auth(delete("/api/workspace/projects/" + code), token)); + mvc.perform(auth(get(workspace), token)).andExpect(status().isNotFound()); + mvc.perform(get(community)).andExpect(status().isNotFound()); + } + + @Test + void rejectsInvalidFormatsAndCommunityShaIncludingEmptyValues() throws Exception { + String token = login("archive-validation@example.invalid"); + String code = create(token); + String workspace = "/api/workspace/projects/" + code + "/archive"; + String community = "/api/community/projects/" + code + "/archive"; + for (String format : java.util.List.of("", "ZIP", "bundle", "../zip")) + mvc.perform(auth(get(workspace).param("format", format), token)).andExpect(status().isBadRequest()); + mvc.perform(auth(get(workspace).param("commitSha", "main"), token)).andExpect(status().isBadRequest()); + for (String sha : java.util.List.of("", "a".repeat(40))) + mvc.perform(get(community).param("commitSha", sha)).andExpect(status().isBadRequest()); + mvc.perform(auth(get(workspace).param("commitSha", "f".repeat(40)), token)).andExpect(status().isNotFound()); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void streamFailureResetsUncommittedAttachmentsButNeverAppendsJsonAfterCommit(boolean commit) throws Exception { + String token = login("archive-failure@example.invalid"); + String code = create(token); + AtomicBoolean closed = new AtomicBoolean(); + InputStream failing = new InputStream() { + private int count; + @Override public int read() { throw new UnsupportedOperationException(); } + @Override public int read(byte[] bytes, int offset, int length) { + if (count == 0) { + byte[] prefix = {'P', 'K', 3, 4}; + System.arraycopy(prefix, 0, bytes, offset, 4); + count += 4; + return 4; + } + if (commit && count == 4) { + int size = Math.min(length, 16384); + java.util.Arrays.fill(bytes, offset, offset + size, (byte) 'x'); + count += size; + return size; + } + throw new GiteaFailure(504, "ARCHIVE_TIMEOUT", "timeout", false); + } + @Override public void close() { closed.set(true); } + }; + doReturn(new GiteaClient.Archive(failing, -1)).when(gitea) + .archive(any(), anyString(), any(), any(), anyLong()); + MvcResult result = mvc.perform(auth(get("/api/workspace/projects/" + code + "/archive"), token)).andReturn(); + assertThat(closed.get()).isTrue(); + if (commit) { + assertThat(result.getResponse().getStatus()).isEqualTo(200); + assertThat(result.getResponse().getContentAsString()).startsWith("PK").doesNotContain("ARCHIVE_TIMEOUT"); + } else { + assertThat(result.getResponse().getStatus()).isEqualTo(504); + assertThat(result.getResponse().getHeader("Content-Disposition")).isNull(); + assertThat(result.getResponse().getHeader("Content-Length")).isNull(); + assertThat(json.readTree(result.getResponse().getContentAsByteArray()).path("code").asText()) + .isEqualTo("ARCHIVE_TIMEOUT"); + } + } +} diff --git a/vibemake-api/src/test/java/com/linkerbot/vibemake/api/FakeGiteaClient.java b/vibemake-api/src/test/java/com/linkerbot/vibemake/api/FakeGiteaClient.java index 4507dcd..cb024cb 100644 --- a/vibemake-api/src/test/java/com/linkerbot/vibemake/api/FakeGiteaClient.java +++ b/vibemake-api/src/test/java/com/linkerbot/vibemake/api/FakeGiteaClient.java @@ -3,7 +3,12 @@ package com.linkerbot.vibemake.api; import com.linkerbot.vibemake.bean.Contracts.*; import com.linkerbot.vibemake.common.PageResponse; import com.linkerbot.vibemake.service.gitea.*; +import com.linkerbot.vibemake.service.archive.ArchiveFormat; +import com.linkerbot.vibemake.testing.ArchiveFixtures; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.time.Duration; import java.security.MessageDigest; import java.util.*; import java.util.concurrent.atomic.AtomicLong; @@ -93,6 +98,19 @@ final class FakeGiteaClient implements GiteaClient { return repo(repository).head; } + @Override + public synchronized Archive archive(Repository repository, String sha, ArchiveFormat format, + Duration timeout, long maxBytes) { + try { + byte[] bytes = ArchiveFixtures.create(format.extension(), snapshot(repository, sha)); + if (bytes.length > maxBytes) + throw new GiteaFailure(413, "ARCHIVE_TOO_LARGE", "归档过大", false); + return new Archive(new ByteArrayInputStream(bytes), bytes.length); + } catch (IOException error) { + throw new IllegalStateException(error); + } + } + @Override public synchronized List tree(Repository repository, String sha, String path) { Map snapshot = repo(repository).snapshots.get(sha); diff --git a/vibemake-api/src/test/java/com/linkerbot/vibemake/api/GiteaLiveTest.java b/vibemake-api/src/test/java/com/linkerbot/vibemake/api/GiteaLiveTest.java index 83147f5..c649228 100644 --- a/vibemake-api/src/test/java/com/linkerbot/vibemake/api/GiteaLiveTest.java +++ b/vibemake-api/src/test/java/com/linkerbot/vibemake/api/GiteaLiveTest.java @@ -11,6 +11,7 @@ import com.linkerbot.vibemake.service.config.VibemakeProperties; import com.linkerbot.vibemake.service.fork.ForkDeadline; import com.linkerbot.vibemake.service.fork.GitCommandRunner; import com.linkerbot.vibemake.service.gitea.GiteaClient; +import com.linkerbot.vibemake.testing.ArchiveFixtures; import org.junit.jupiter.api.*; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; @@ -143,6 +144,55 @@ class GiteaLiveTest { .asText(); } + @Test + void realRepositoryArchivesKeepPublishedVersionAndBothFormats() throws Exception { + String token = login("live-archive@example.invalid"); + String code = perform(body(auth(post("/api/workspace/projects"), token), + Map.of("name", "仓库归档联调", "projectType", "GENERAL")), 200).path("projectCode").asText(); + String original = perform(auth(get("/api/workspace/projects/" + code + "/tree"), token), 200) + .path("commitSha").asText(); + String pointer = "version https://git-lfs.github.com/spec/v1\noid sha256:" + "a".repeat(64) + "\nsize 123\n"; + String first = perform(body(auth(post("/api/workspace/projects/" + code + "/commits"), token), + Map.of("expectedCommitSha", original, "message", "发布归档 A", "changes", List.of( + Map.of("action", "UPDATE_FILE", "path", "README.md", "content", "版本 A"), + Map.of("action", "CREATE_FILE", "path", "目录/中文.txt", "content", "中文内容"), + Map.of("action", "CREATE_FILE", "path", "model.lfs", "content", pointer)))), 200) + .path("commitSha").asText(); + byte[] binary = new byte[] {0, 1, -1, 2}; + first = perform(auth(multipart("/api/workspace/projects/" + code + "/uploads/commit") + .file(new MockMultipartFile("files", "mesh.bin", "application/octet-stream", binary)) + .param("expectedCommitSha", first).param("message", "添加归档二进制"), token), 200) + .path("commitSha").asText(); + perform(body(auth(put("/api/workspace/projects/" + code + "/publication"), token), + Map.of("commitSha", first)), 200); + String second = perform(body(auth(post("/api/workspace/projects/" + code + "/commits"), token), + Map.of("expectedCommitSha", first, "message", "未发布归档 B", "changes", List.of( + Map.of("action", "UPDATE_FILE", "path", "README.md", "content", "版本 B")))), 200) + .path("commitSha").asText(); + for (String format : List.of("zip", "tar.gz")) { + for (String context : List.of("community", "workspace", "history")) { + boolean workspace = !context.equals("community"); + String expected = context.equals("workspace") ? second : first; + MockHttpServletRequestBuilder request = get("/api/" + (workspace ? "workspace" : "community") + + "/projects/" + code + "/archive").param("format", format); + if (workspace) auth(request, token); + if (context.equals("history")) request.param("commitSha", first); + MvcResult result = mvc.perform(request).andExpect(status().isOk()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(header().string("Content-Disposition", org.hamcrest.Matchers.containsString( + code + "-" + expected.substring(0, 12) + "." + format))).andReturn(); + Map files = ArchiveFixtures.read(format, result.getResponse().getContentAsByteArray()); + String root = files.keySet().stream().filter(name -> name.endsWith("README.md")) + .findFirst().orElseThrow().replace("README.md", ""); + assertThat(new String(files.get(root + "README.md"), StandardCharsets.UTF_8)) + .isEqualTo(context.equals("workspace") ? "版本 B" : "版本 A"); + assertThat(files.get(root + "mesh.bin")).isEqualTo(binary); + assertThat(new String(files.get(root + "目录/中文.txt"), StandardCharsets.UTF_8)).isEqualTo("中文内容"); + assertThat(new String(files.get(root + "model.lfs"), StandardCharsets.UTF_8)).isEqualTo(pointer); + } + } + } + @Test void realSnapshotForkPreservesLargeCadGitlinksSymlinksAndLfsWithoutUnpublishedHistory() throws Exception { diff --git a/vibemake-common/src/test/java/com/linkerbot/vibemake/testing/ArchiveFixtures.java b/vibemake-common/src/test/java/com/linkerbot/vibemake/testing/ArchiveFixtures.java new file mode 100644 index 0000000..5fe1590 --- /dev/null +++ b/vibemake-common/src/test/java/com/linkerbot/vibemake/testing/ArchiveFixtures.java @@ -0,0 +1,94 @@ +package com.linkerbot.vibemake.testing; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +/** 小型归档测试数据;不进入生产制品。TAR 仅生成测试使用的短文件路径。 */ +public final class ArchiveFixtures { + private ArchiveFixtures() {} + + public static byte[] create(String format, Map files) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + if (format.equals("zip")) { + try (ZipOutputStream zip = new ZipOutputStream(bytes)) { + for (Map.Entry file : files.entrySet()) { + zip.putNextEntry(new ZipEntry(file.getKey())); + zip.write(file.getValue()); + zip.closeEntry(); + } + } + } else { + try (GZIPOutputStream gzip = new GZIPOutputStream(bytes)) { + for (Map.Entry file : files.entrySet()) { + byte[] header = new byte[512]; + put(header, 0, file.getKey()); + put(header, 100, "0000644"); + put(header, 108, "0000000"); + put(header, 116, "0000000"); + put(header, 124, String.format("%011o", file.getValue().length)); + put(header, 136, "00000000000"); + Arrays.fill(header, 148, 156, (byte) ' '); + header[156] = '0'; + put(header, 257, "ustar"); + put(header, 263, "00"); + int checksum = 0; + for (byte value : header) checksum += value & 0xff; + put(header, 148, String.format("%06o\0 ", checksum)); + gzip.write(header); + gzip.write(file.getValue()); + gzip.write(new byte[(512 - file.getValue().length % 512) % 512]); + } + gzip.write(new byte[1024]); + } + } + return bytes.toByteArray(); + } + + private static void put(byte[] header, int offset, String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + System.arraycopy(bytes, 0, header, offset, bytes.length); + } + + public static Map read(String format, byte[] bytes) throws IOException { + Map files = new LinkedHashMap<>(); + if (format.equals("zip")) { + try (ZipInputStream zip = new ZipInputStream(new ByteArrayInputStream(bytes))) { + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) + if (!entry.isDirectory()) files.put(entry.getName(), zip.readAllBytes()); + } + } else { + try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(bytes))) { + byte[] header; + while ((header = gzip.readNBytes(512)).length == 512 && header[0] != 0) { + String name = text(header, 0, 100); + int size = Integer.parseInt(text(header, 124, 12).trim(), 8); + byte[] content = gzip.readNBytes(size); + if (content.length != size) throw new IOException("TAR 数据被截断"); + if (header[156] == '0' || header[156] == 0) files.put(name, content); + gzip.skipNBytes((512 - size % 512) % 512); + } + if (header.length != 512) throw new IOException("TAR 缺少结束块"); + gzip.transferTo(OutputStream.nullOutputStream()); // 读到末尾,校验 gzip CRC 和长度。 + } + } + return files; + } + + private static String text(byte[] bytes, int offset, int maximum) { + int end = offset; + while (end < offset + maximum && bytes[end] != 0) end++; + return new String(bytes, offset, end - offset, StandardCharsets.UTF_8); + } +} diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/config/LimitProperties.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/config/LimitProperties.java index e45ccb7..061bbf0 100644 --- a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/config/LimitProperties.java +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/config/LimitProperties.java @@ -35,6 +35,7 @@ public class LimitProperties { values.put("admin-login", 30); values.put("vibemake-read", 120); values.put("file-read", 60); + values.put("archive-read", 10); values.put("create", 5); values.put("fork", 10); values.put("file-write", 10); diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java index a01d813..0b8b685 100644 --- a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java @@ -2,7 +2,10 @@ package com.linkerbot.vibemake.service.gitea; import com.linkerbot.vibemake.bean.Contracts.*; import com.linkerbot.vibemake.common.PageResponse; +import com.linkerbot.vibemake.service.archive.ArchiveFormat; +import java.io.IOException; +import java.io.InputStream; import java.util.List; import java.time.Duration; @@ -13,6 +16,16 @@ public interface GiteaClient { record StoredFile(String path, String blobSha, byte[] bytes) {} + record Archive(InputStream input, long contentLength) implements AutoCloseable { + @Override + public void close() throws IOException { input.close(); } + } + + default Archive archive(Repository repository, String sha, ArchiveFormat format, + Duration timeout, long maxBytes) { + throw new UnsupportedOperationException("未实现仓库归档下载"); + } + record Change(String operation, String path, String blobSha, byte[] bytes) {} /** 只从 main 可达的提交中查找,并校验完整尾注及唯一父提交;空结果不是失败证明。 */ @@ -62,6 +75,8 @@ public interface GiteaClient { String head(Repository repository); + default String head(Repository repository, Duration timeout) { return head(repository); } + List tree(Repository repository, String sha, String path); StoredFile file(Repository repository, String sha, String path); diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java index c565007..587400a 100644 --- a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java @@ -9,15 +9,18 @@ import com.linkerbot.vibemake.common.LimitedHttpBody; import com.linkerbot.vibemake.common.PageResponse; import com.linkerbot.vibemake.service.config.VibemakeProperties; import com.linkerbot.vibemake.service.fork.ForkDeadline; +import com.linkerbot.vibemake.service.archive.ArchiveFormat; import org.springframework.stereotype.Component; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URLEncoder; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.net.http.HttpTimeoutException; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.*; @@ -53,6 +56,47 @@ public class RestGiteaClient implements GiteaClient { .orElse(""); } + @Override + public Archive archive(Repository repository, String sha, ArchiveFormat format, + Duration timeout, long maxBytes) { + if (properties.getGitea().getToken().isBlank()) + throw new GiteaFailure(503, "GITEA_NOT_CONFIGURED", "请配置 Gitea Token", false); + long deadline = System.nanoTime() + timeout.toNanos(); + String base = properties.getGitea().getBaseUrl().replaceAll("/+$", ""); + HttpRequest request = HttpRequest.newBuilder(URI.create(base + "/api/v1" + + repoPath(repository) + "/archive/" + FilePaths.sha(sha) + "." + format.extension())) + .timeout(timeout) + .header("Accept", format.contentType()) + .header("Authorization", "token " + properties.getGitea().getToken()) + .GET().build(); + try { + HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofInputStream()); + InputStream input = response.body(); + try { + if (response.statusCode() == 404) + throw new GiteaFailure(404, "NOT_FOUND", "仓库或提交版本不存在", false); + if (response.statusCode() != 200) throw ArchiveInputStream.failed(); + long length = response.headers().firstValueAsLong("Content-Length").orElse(-1); + if (length < -1 || length == 0) throw ArchiveInputStream.failed(); + if (length > maxBytes) throw ArchiveInputStream.tooLarge(); + return new Archive(new ArchiveInputStream(input, length, maxBytes, deadline), length); + } catch (RuntimeException error) { + try { input.close(); } catch (IOException ignored) { } + if (error instanceof NumberFormatException) throw ArchiveInputStream.failed(); + throw error; + } + } catch (HttpTimeoutException error) { + throw ArchiveInputStream.timedOut(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw ArchiveInputStream.timedOut(); + } catch (IOException error) { + GiteaFailure failure = ArchiveInputStream.failed(); + failure.initCause(error); + throw failure; + } + } + private HttpResponse request(String method, String path, Object body) { return request(method, path, body, properties.getGitea().getRequestTimeout()); } @@ -156,6 +200,22 @@ public class RestGiteaClient implements GiteaClient { return branch(repo, "main"); } + @Override + public String head(Repository repo, Duration timeout) { + try { + JsonNode branch = parse(request("GET", repoPath(repo) + "/branches/main", null, timeout).body(), false); + return FilePaths.sha(branch.path("commit").path("id").asText()); + } catch (GiteaFailure error) { + if (error.status() == 504) throw ArchiveInputStream.timedOut(); + if (error.status() == 404 || error.status() == 503) throw error; + GiteaFailure failure = ArchiveInputStream.failed(); + failure.initCause(error); + throw failure; + } catch (BusinessException error) { + throw ArchiveInputStream.failed(); + } + } + @Override public String branch(Repository repo, String branchName) { JsonNode branch = diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/state/HttpSafetyFilter.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/state/HttpSafetyFilter.java index c641182..1b43765 100644 --- a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/state/HttpSafetyFilter.java +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/state/HttpSafetyFilter.java @@ -67,6 +67,8 @@ public class HttpSafetyFilter extends OncePerRequestFilter { if (path.equals("/api/auth/captcha")) limits.request("captcha", ip); else if (path.equals("/api/auth/login")) limits.request("login-ip", ip); else if (path.equals("/admin-api/auth/login")) limits.request("admin-login", ip); + else if (read && path.matches("/api/(community|workspace)/projects/[^/]+/archive")) + limits.request("archive-read", ip); else if (read && path.matches( "/(api/community|api/workspace|admin-api)/projects/[^/]+/files")) diff --git a/vibemake-service/src/test/java/com/linkerbot/vibemake/service/archive/ArchiveServiceTest.java b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/archive/ArchiveServiceTest.java new file mode 100644 index 0000000..8813bb2 --- /dev/null +++ b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/archive/ArchiveServiceTest.java @@ -0,0 +1,137 @@ +package com.linkerbot.vibemake.service.archive; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import com.linkerbot.vibemake.bean.entity.Project; +import com.linkerbot.vibemake.common.BusinessException; +import com.linkerbot.vibemake.service.ProjectService; +import com.linkerbot.vibemake.service.config.ArchiveProperties; +import com.linkerbot.vibemake.service.gitea.GiteaClient; +import com.linkerbot.vibemake.service.gitea.GiteaFailure; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +class ArchiveServiceTest { + private static final String CODE = "111111111111"; + private static final String SHA = "a".repeat(40); + private static final byte[] PREFIX = {'P', 'K', 3, 4, 0, 0}; + private final GiteaClient gitea = mock(GiteaClient.class); + private final ProjectService projects = mock(ProjectService.class); + private ArchiveService service; + + @BeforeEach + void setup() { + Project project = new Project(); + project.setProjectCode(CODE); + project.setPublishedCommitSha(SHA); + when(projects.published(CODE)).thenReturn(project); + when(projects.readyOwner(1, CODE)).thenReturn(project); + when(projects.repository(project)).thenReturn(new GiteaClient.Repository(1, "vibemake", "repo")); + when(gitea.head(any(), any())).thenReturn(SHA); + ArchiveProperties properties = new ArchiveProperties(); + properties.setMaxConcurrent(1); + service = new ArchiveService(projects, gitea, properties); + } + + private void healthy() { + when(gitea.archive(any(), anyString(), any(), any(), anyLong())).thenAnswer(call -> + new GiteaClient.Archive(new ByteArrayInputStream(PREFIX), PREFIX.length)); + } + + @Test + void concurrentDownloadRejectsWithoutBlockingAndReleasesOnCompletion() throws Exception { + healthy(); + CountDownLatch opened = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + Future first = executor.submit(() -> { + service.community(CODE, null, (name, type, length) -> { + opened.countDown(); + try { + if (!release.await(3, TimeUnit.SECONDS)) throw new IOException("test timeout"); + } catch (InterruptedException error) { throw new IOException(error); } + return OutputStream.nullOutputStream(); + }); + return null; + }); + try { + assertThat(opened.await(2, TimeUnit.SECONDS)).isTrue(); + assertThatThrownBy(() -> service.community(CODE, null, + (name, type, length) -> OutputStream.nullOutputStream())) + .isInstanceOfSatisfying(BusinessException.class, + error -> assertThat(error.code()).isEqualTo("ARCHIVE_BUSY")); + verify(gitea, times(1)).archive(any(), anyString(), any(), any(), anyLong()); + } finally { release.countDown(); } + first.get(3, TimeUnit.SECONDS); + } + service.community(CODE, null, (name, type, length) -> OutputStream.nullOutputStream()); + } + + @ParameterizedTest + @ValueSource(strings = {"ARCHIVE_TIMEOUT", "ARCHIVE_TOO_LARGE", "GITEA_ERROR", "client"}) + void closesUpstreamAndReleasesPermitAfterReadOrClientFailure(String failure) throws Exception { + AtomicBoolean closed = new AtomicBoolean(); + InputStream input = new ByteArrayInputStream(PREFIX) { + @Override + public synchronized int read(byte[] bytes, int offset, int length) { + if (pos >= 4 && !failure.equals("client")) + throw new GiteaFailure(502, failure, "fixture", false); + return super.read(bytes, offset, length); + } + + @Override + public void close() { closed.set(true); } + }; + when(gitea.archive(any(), anyString(), any(), any(), anyLong())) + .thenReturn(new GiteaClient.Archive(input, -1)); + assertThatThrownBy(() -> service.community(CODE, null, (name, type, length) -> + failure.equals("client") ? new OutputStream() { + @Override public void write(int value) throws IOException { throw new IOException("disconnected"); } + } : OutputStream.nullOutputStream())) + .isInstanceOf(failure.equals("client") ? IOException.class : GiteaFailure.class); + assertThat(closed.get()).isTrue(); + healthy(); + service.community(CODE, null, (name, type, length) -> OutputStream.nullOutputStream()); + } + + @Test + void invalidUpstreamContentNeverOpensAttachmentAndDoesNotLeakPermit() throws Exception { + when(gitea.archive(any(), anyString(), any(), any(), anyLong())).thenReturn( + new GiteaClient.Archive(new ByteArrayInputStream("login".getBytes()), -1)); + assertThatThrownBy(() -> service.community(CODE, "zip", (name, type, length) -> { + fail("无效归档不能发出附件响应"); + return OutputStream.nullOutputStream(); + })).isInstanceOfSatisfying(GiteaFailure.class, + error -> assertThat(error.code()).isEqualTo("GITEA_ERROR")); + healthy(); + service.community(CODE, null, (name, type, length) -> OutputStream.nullOutputStream()); + } + + @Test + void frozenShaAndFilenameRemainStableIfHeadChanges() throws Exception { + healthy(); + service.owner(1, CODE, "zip", null, (name, type, length) -> { + when(gitea.head(any(), any())).thenReturn("b".repeat(40)); + assertThat(name).isEqualTo(CODE + "-aaaaaaaaaaaa.zip"); + assertThat(type).isEqualTo("application/zip"); + return OutputStream.nullOutputStream(); + }); + verify(gitea, times(1)).head(any(), any()); + verify(gitea).archive(any(), eq(SHA), eq(ArchiveFormat.ZIP), any(), anyLong()); + } +} diff --git a/vibemake-service/src/test/java/com/linkerbot/vibemake/service/gitea/GiteaArchiveTest.java b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/gitea/GiteaArchiveTest.java new file mode 100644 index 0000000..6a8024d --- /dev/null +++ b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/gitea/GiteaArchiveTest.java @@ -0,0 +1,204 @@ +package com.linkerbot.vibemake.service.gitea; + +import static org.assertj.core.api.Assertions.*; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkerbot.vibemake.service.archive.ArchiveFormat; +import com.linkerbot.vibemake.service.config.VibemakeProperties; +import com.linkerbot.vibemake.testing.ArchiveFixtures; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Timeout(15) +class GiteaArchiveTest { + private static final String SHA = "a".repeat(40); + private static final GiteaClient.Repository REPO = new GiteaClient.Repository(1, "vibemake", "repo"); + private HttpServer server; + private ExecutorService workers; + private RestGiteaClient client; + + @BeforeEach + void start() throws Exception { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + workers = Executors.newVirtualThreadPerTaskExecutor(); + server.setExecutor(workers); + server.start(); + VibemakeProperties config = new VibemakeProperties(); + config.getGitea().setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort()); + config.getGitea().setToken("fixture-token"); + config.setMaxFileBytes(1); // 归档必须独立于普通单文件读取上限。 + client = new RestGiteaClient(config, new ObjectMapper()); + } + + @AfterEach + void stop() { + server.stop(0); + workers.shutdownNow(); + } + + private String path(ArchiveFormat format) { return "/api/v1/repos/vibemake/repo/archive/" + SHA + "." + format.extension(); } + + @ParameterizedTest + @EnumSource(ArchiveFormat.class) + void retrievesFrozenVersionWithServerCredentialsAndCorrectContents(ArchiveFormat format) throws Exception { + Map files = Map.of("目录/底盘.bin", new byte[] {0, 1, -1}, + "README.md", "中文说明".getBytes(StandardCharsets.UTF_8)); + byte[] bytes = ArchiveFixtures.create(format.extension(), files); + AtomicInteger calls = new AtomicInteger(); + server.createContext(path(format), exchange -> { + assertThat(exchange.getRequestHeaders().getFirst("Authorization")).isEqualTo("token fixture-token"); + assertThat(exchange.getRequestHeaders().getFirst("Accept")).isEqualTo(format.contentType()); + calls.incrementAndGet(); + exchange.sendResponseHeaders(200, bytes.length); + try (OutputStream output = exchange.getResponseBody()) { output.write(bytes); } + }); + try (GiteaClient.Archive archive = client.archive(REPO, SHA, format, Duration.ofSeconds(3), 10000)) { + assertThat(archive.contentLength()).isEqualTo(bytes.length); + Map actual = ArchiveFixtures.read(format.extension(), archive.input().readAllBytes()); + assertThat(actual.keySet()).isEqualTo(files.keySet()); + for (String name : files.keySet()) assertThat(actual.get(name)).isEqualTo(files.get(name)); + } + assertThat(calls.get()).isEqualTo(1); + } + + @ParameterizedTest + @ValueSource(ints = {302, 401, 403, 404, 500}) + void upstreamErrorsAndRedirectsAreNotPassedThrough(int status) { + AtomicInteger redirected = new AtomicInteger(); + server.createContext("/redirected", exchange -> { redirected.incrementAndGet(); exchange.close(); }); + server.createContext(path(ArchiveFormat.ZIP), exchange -> { + exchange.getResponseHeaders().set("Location", "/redirected"); + exchange.sendResponseHeaders(status, -1); + exchange.close(); + }); + assertThatThrownBy(() -> client.archive(REPO, SHA, ArchiveFormat.ZIP, Duration.ofSeconds(2), 100)) + .isInstanceOfSatisfying(GiteaFailure.class, error -> { + assertThat(error.code()).isEqualTo(status == 404 ? "NOT_FOUND" : "GITEA_ERROR"); + assertThat(error.uncertain()).isFalse(); + }); + assertThat(redirected.get()).isZero(); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void declaredAndChunkedArchivesAreBounded(boolean declared) throws Exception { + server.createContext(path(ArchiveFormat.ZIP), exchange -> { + exchange.sendResponseHeaders(200, declared ? 10000 : 0); + try (OutputStream output = exchange.getResponseBody()) { output.write(new byte[10000]); } + catch (java.io.IOException expected) { /* 超限后关闭连接。 */ } + }); + assertThatThrownBy(() -> { + try (GiteaClient.Archive archive = client.archive(REPO, SHA, ArchiveFormat.ZIP, Duration.ofSeconds(2), 100)) { + archive.input().transferTo(OutputStream.nullOutputStream()); + } + }).isInstanceOfSatisfying(GiteaFailure.class, + error -> assertThat(error.code()).isEqualTo("ARCHIVE_TOO_LARGE")); + } + + @Test + void streamsBeyondFiftyMiBWithoutRequiringWholeBody() throws Exception { + int total = 51 * 1024 * 1024; + CountDownLatch readerStarted = new CountDownLatch(1); + server.createContext(path(ArchiveFormat.ZIP), exchange -> { + exchange.sendResponseHeaders(200, 0); + try (OutputStream output = exchange.getResponseBody()) { + output.write(new byte[1024]); + output.flush(); + if (!readerStarted.await(3, TimeUnit.SECONDS)) throw new AssertionError("不能预先读取完整归档"); + byte[] block = new byte[64 * 1024]; + for (int written = 1024; written < total; written += block.length) + output.write(block, 0, Math.min(block.length, total - written)); + } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + }); + try (GiteaClient.Archive archive = client.archive(REPO, SHA, ArchiveFormat.ZIP, Duration.ofSeconds(8), total)) { + assertThat(archive.input().read()).isZero(); + readerStarted.countDown(); + assertThat(archive.input().transferTo(OutputStream.nullOutputStream())).isEqualTo(total - 1L); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void timeoutCoversHeadersAndStalledBody(boolean headersSent) throws Exception { + CountDownLatch release = new CountDownLatch(1); + server.createContext(path(ArchiveFormat.ZIP), exchange -> { + try { + if (headersSent) { + exchange.sendResponseHeaders(200, 0); + exchange.getResponseBody().write(new byte[] {'P', 'K', 3, 4}); + exchange.getResponseBody().flush(); + } + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + finally { exchange.close(); } + }); + long start = System.nanoTime(); + try { + assertThatThrownBy(() -> { + try (GiteaClient.Archive archive = client.archive(REPO, SHA, ArchiveFormat.ZIP, + Duration.ofMillis(400), 100)) { + archive.input().transferTo(OutputStream.nullOutputStream()); + } + }).isInstanceOfSatisfying(GiteaFailure.class, + error -> assertThat(error.code()).isEqualTo("ARCHIVE_TIMEOUT")); + assertThat(Duration.ofNanos(System.nanoTime() - start)).isLessThan(Duration.ofSeconds(3)); + } finally { release.countDown(); } + } + + @Test + void shortKnownLengthResponseIsAnError() { + server.createContext(path(ArchiveFormat.ZIP), exchange -> { + exchange.sendResponseHeaders(200, 100); + exchange.getResponseBody().write(new byte[4]); + exchange.close(); + }); + assertThatThrownBy(() -> { + try (GiteaClient.Archive archive = client.archive(REPO, SHA, ArchiveFormat.ZIP, Duration.ofSeconds(2), 1000)) { + archive.input().transferTo(OutputStream.nullOutputStream()); + } + }).isInstanceOfSatisfying(GiteaFailure.class, + error -> assertThat(error.code()).isEqualTo("GITEA_ERROR")); + } + + @ParameterizedTest + @ValueSource(strings = {"invalid-json", "invalid-sha", "timeout"}) + void freezingMainUsesArchiveBudgetAndReadErrorCodes(String scenario) { + CountDownLatch release = new CountDownLatch(1); + server.createContext("/api/v1/repos/vibemake/repo/branches/main", exchange -> { + try { + if (scenario.equals("timeout")) release.await(3, TimeUnit.SECONDS); + else { + byte[] body = (scenario.equals("invalid-json") ? "invalid-json" : "{\"commit\":{\"id\":\"main\"}}") + .getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + } + } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + finally { exchange.close(); } + }); + try { + assertThatThrownBy(() -> client.head(REPO, Duration.ofMillis(300))) + .isInstanceOfSatisfying(GiteaFailure.class, error -> { + assertThat(error.code()).isEqualTo(scenario.equals("timeout") ? "ARCHIVE_TIMEOUT" : "GITEA_ERROR"); + assertThat(error.uncertain()).isFalse(); + }); + } finally { release.countDown(); } + } +} diff --git a/vibemake-service/src/test/java/com/linkerbot/vibemake/service/state/SafetyTest.java b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/state/SafetyTest.java index 4feb33a..ef11117 100644 --- a/vibemake-service/src/test/java/com/linkerbot/vibemake/service/state/SafetyTest.java +++ b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/state/SafetyTest.java @@ -18,6 +18,27 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; class SafetyTest { + @Test + void archiveQuotaIsSharedAcrossEntrypointsAndIndependentOfFilePermits() throws Exception { + LimitProperties properties = new LimitProperties(); + properties.getPerMinute().put("archive-read", 1); + properties.setFileConcurrent(1); + HttpSafetyFilter filter = new HttpSafetyFilter(properties, + new RateLimits(new InMemoryStateStore(), properties), new ClientAddresses(properties), new ObjectMapper()); + filter.doFilter(new MockHttpServletRequest("GET", "/api/community/projects/111111111111/files"), + new MockHttpServletResponse(), (request, response) -> { + MockHttpServletResponse archive = new MockHttpServletResponse(); + filter.doFilter(new MockHttpServletRequest("GET", "/api/community/projects/111111111111/archive"), + archive, (inner, output) -> {}); + assertThat(archive.getStatus()).isEqualTo(200); + }); + MockHttpServletResponse limited = new MockHttpServletResponse(); + filter.doFilter(new MockHttpServletRequest("GET", "/api/workspace/projects/111111111111/archive"), + limited, (request, response) -> fail("工作区和社区应共用归档额度")); + assertThat(limited.getStatus()).isEqualTo(429); + assertThat(limited.getHeader("Retry-After")).isNotBlank(); + } + @org.junit.jupiter.params.ParameterizedTest @org.junit.jupiter.params.provider.ValueSource( strings = {