From 366426d98a5a6e673ba89c29fdf8367e75e05bbf Mon Sep 17 00:00:00 2001 From: lxp <2770281812@qq.com> Date: Thu, 5 Nov 2026 15:40:02 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8D=95=E6=96=87=E4=BB=B6=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E6=B5=81=E5=BC=8F=E4=BC=A0=E8=BE=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +- docs/admin-api.md | 9 +- docs/admin-deployment.md | 8 +- docs/frontend-repository-download-git.md | 2 +- docs/vibemake-api.md | 21 +- .../controller/AdminProjectController.java | 23 +- .../src/main/resources/application.yml | 3 + .../vibemake/adminapi/AdminWorkflowTest.java | 39 +++ .../vibemake/adminapi/FakeGiteaClient.java | 12 + .../api/controller/VibemakeController.java | 15 +- .../api/controller/WorkspaceController.java | 25 +- .../src/main/resources/application.yml | 3 + .../vibemake/api/FakeGiteaClient.java | 10 + .../vibemake/api/FileDownloadApiTest.java | 244 ++++++++++++++++ .../config/FileDownloadProperties.java | 27 ++ .../download/FileDownloadResponse.java | 57 ++++ .../service/download/FileDownloadService.java | 95 ++++++ .../gitea/FileDownloadInputStream.java | 102 +++++++ .../vibemake/service/gitea/GiteaClient.java | 16 +- .../service/gitea/RestGiteaClient.java | 87 +++++- .../download/FileDownloadServiceTest.java | 79 +++++ .../service/gitea/GiteaFileDownloadTest.java | 271 ++++++++++++++++++ 22 files changed, 1101 insertions(+), 51 deletions(-) create mode 100644 vibemake-api/src/test/java/com/linkerbot/vibemake/api/FileDownloadApiTest.java create mode 100644 vibemake-service/src/main/java/com/linkerbot/vibemake/service/config/FileDownloadProperties.java create mode 100644 vibemake-service/src/main/java/com/linkerbot/vibemake/service/download/FileDownloadResponse.java create mode 100644 vibemake-service/src/main/java/com/linkerbot/vibemake/service/download/FileDownloadService.java create mode 100644 vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/FileDownloadInputStream.java create mode 100644 vibemake-service/src/test/java/com/linkerbot/vibemake/service/download/FileDownloadServiceTest.java create mode 100644 vibemake-service/src/test/java/com/linkerbot/vibemake/service/gitea/GiteaFileDownloadTest.java diff --git a/README.md b/README.md index 5dc640d..4b6ef22 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,9 @@ python3 scripts/test-ai.py 服务账号的 GITEA_TOKEN 需同时允许 REST 仓库创建、HTTP Git 读写及 LFS 读写。凭据通过进程环境注入,只用于配置的 Gitea 地址;不写入 URL、命令参数、Git 配置或日志。Fork 接口继续同步返回,前端和反向代理读取超时至少400秒;AI 进入 forking 后切换专用预算并继续发送心跳。 -Fork 不受普通在线文件读取的 50 MiB 限制,文件浏览、下载和 LFS 在线展开能力仍维持原规则。失败时先核对响应 data.projectCode:明确失败为 FAILED,远端写入或数据库状态待核对为 CREATING;不会自动重发、删除远端仓库或 OSS 对象。完成后清理本地临时目录。使用 scripts/test-gitea.py 可执行显式真实 Gitea 联调,仅创建并清理本次测试专用仓库,不修改已有社区项目。 +Fork 不受 JSON 文件预览的 50 MiB 限制或单文件流式下载的 1 GiB 限制。Fork 失败时先核对响应 data.projectCode:明确失败为 FAILED,远端写入或数据库状态待核对为 CREATING;不会自动重发、删除远端仓库或 OSS 对象。完成后清理本地临时目录。使用 scripts/test-gitea.py 可执行显式真实 Gitea 联调,仅创建并清理本次测试专用仓库,不修改已有社区项目。 + +JSON 预览保留原有上限;三个端的 `download=true` 使用流式传输,默认单文件最大 1 GiB、上游处理和读取预算 5 分钟,分别由 `FILE_DOWNLOAD_MAX_BYTES`、`FILE_DOWNLOAD_TIMEOUT` 配置,LFS 指针不在线展开。下载路径需关闭代理响应缓冲并为代理和客户端至少预留 400 秒超时,详见[文件接口](docs/vibemake-api.md#目录和文件读取)。 ## 运行约定 diff --git a/docs/admin-api.md b/docs/admin-api.md index 60cfb93..193a37e 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -629,7 +629,11 @@ async function readProjectFile(projectCode: string, path: string, commitSha?: st `encoding=utf-8` 且 `editable=true` 才开放文本编辑;`base64` 代表原始文件内容的 Base64,可按文件类型预览或下载,不能直接当文本提交。在线文本上限默认 1 MiB。符号链接、子模块不支持按普通文件读取,迁移仓库中的 LFS 文件也不保证能在线预览。 -下载使用携带后台 Authorization 的 `fetch`,成功后 `response.blob()`,从 `Content-Disposition` 获取文件名并触发保存。不要把 Token 拼到 URL,也不要直接用不带鉴权的 `` 下载。下载失败仍可能是 JSON 错误,应先判断 HTTP 状态。 +JSON 文件预览默认最大 50 MiB;`download=true` 使用独立流式下载,默认最大 **1 GiB**,由 `FILE_DOWNLOAD_MAX_BYTES` 配置,上游处理和读取总预算默认 **5 分钟**,由 `FILE_DOWNLOAD_TIMEOUT` 配置。下载按请求受理时的完整 SHA 固定版本,仍支持 `commitSha`。空文件可下载,LFS 指针按仓库原文返回,不展开实体。 + +下载使用携带后台 Authorization 的 `fetch`,成功后 `response.blob()`,从 `Content-Disposition` 获取 UTF-8 文件名并触发保存。成功为 `application/octet-stream`,带 `Content-Length`、`Cache-Control: no-store` 和 `X-Request-Id`。不要把 Token 拼到 URL,也不要直接用不带鉴权的 `` 下载。客户端超时建议至少预留 **400 秒**,可提供取消操作。 + +下载沿用文件读取限流及 `FILE_MAX_CONCURRENT` 名额,名额持有到传输结束;超限为 `413 FILE_TOO_LARGE`,超时为 `504 FILE_DOWNLOAD_TIMEOUT`。发包前失败返回 JSON,应先判断 HTTP 状态;传输开始后失败会直接停止响应,可能仍是 HTTP 200,不会在文件尾部追加 JSON。网络错误、取消或长度不符时丢弃残缺文件,由用户手动重试;不提供 Range 断点续传。 ### 6.3 在线文本、多文件变更 @@ -1228,7 +1232,8 @@ async function watchImport( | 409 DUPLICATE_SOURCE | 使用错误 data.projectCode 引导查看已有项目 | | 409 CLEANUP_REPOSITORY_UNSAFE / CLEANUP_REPOSITORY_REPLACED | 残留仓库核验不通过,停止清理;任务中的同名 errorCode 也按此提示 | | REMOTE_RESULT_UNKNOWN(状态码以响应为准) | 写入结果待核对,停止重复提交,展示 requestId | -| 413 FILE_TOO_LARGE / UPLOAD_TOO_LARGE / URDF_UPLOAD_TOO_LARGE | 检查对应上传类型限额,拆批或减少文件 | +| 413 FILE_TOO_LARGE / UPLOAD_TOO_LARGE / URDF_UPLOAD_TOO_LARGE | 检查对应预览、下载或上传大小上限;上传可拆批或减少文件 | +| 499 FILE_DOWNLOAD_CANCELLED / 504 FILE_DOWNLOAD_TIMEOUT | 下载取消或超时,丢弃残缺文件,可手动重试 | | 413 UPLOAD_TOO_MANY_PARTS | 文件或表单项数量超限,减少文件并检查重复添加的字段 | | 429 ADMIN_LOGIN_LIMITED | 提示 15 分钟后重试,不循环发登录请求 | | 503 OSS_UNAVAILABLE | 暂停上传入口并提示当前环境未开放上传 | diff --git a/docs/admin-deployment.md b/docs/admin-deployment.md index e9e5f99..a8c09f3 100644 --- a/docs/admin-deployment.md +++ b/docs/admin-deployment.md @@ -159,13 +159,19 @@ Redis 只监听内网或回环地址,启用认证;跨不可信网络使用 T | HTTP_URDF_JSON_MAX_BYTES | 2097152,URDF 清单 2 MiB | | HTTP_COMMIT_JSON_MAX_BYTES | 67108864,提交 JSON 64 MiB | | FILE_MAX_CONCURRENT | 4,单实例文件读取/目录/历史/提交/multipart 处理名额 | +| FILE_DOWNLOAD_MAX_BYTES | 1073741824,单文件流式下载上限 1 GiB,社区端和管理端均配置 | +| FILE_DOWNLOAD_TIMEOUT | 5m,获取 SHA、文件元数据、等待上游响应及读取响应体的总预算 | | FORK_MAX_CONCURRENT | 2,保持独立 Fork 执行名额 | JSON 上限为实际线上请求字节数,包含转义和结构开销;业务单文件/总内容上限继续保留。Content-Length 和 chunked 均流式核验,超限返回 413 REQUEST_TOO_LARGE。仓库上传单文件最多 50 MiB、文件合计最多 200 MiB、每批最多 500 个文件,multipart 整个请求最多 202 MiB(预留 2 MiB 表单开销),原有 UPLOAD_TOO_LARGE/FILE_TOO_LARGE 错误保留;部署反向代理时同步确保请求体上限不低于 202 MiB。URDF 直传 OSS 不占用这类 multipart 名额。文件名额不足为 429 FILE_BUSY,不排队;Fork 繁忙仍为 FORK_BUSY。 API 和管理端均显式配置 `server.tomcat.max-part-count=1003`,允许 500 个 files、500 个 paths 及 3 个公共字段。Tomcat 默认的 50 项无法承载业务允许的批量上传;数量超限会被 Spring 包装成上传大小异常,现区分为 413 `UPLOAD_TOO_MANY_PARTS` 并记录安全诊断日志。部署时若覆盖该配置,需要与业务批量文件数对齐。 -Gitea 原始文件有界读取,contents JSON 上限按单文件 Base64 膨胀量加 2 MiB 元信息计算,其他元数据 8 MiB,错误响应 64 KiB。GET 超限返回 502 GITEA_RESPONSE_TOO_LARGE;写入响应无法完整确认仍为 REMOTE_RESULT_UNKNOWN,持久化阻断后续仓库写入,不自动重试。 +JSON 预览继续使用 50 MiB 文件读取上限,contents JSON 上限按单文件 Base64 膨胀量加 2 MiB 元信息计算,其他元数据 8 MiB,错误响应 64 KiB。GET 超限返回 502 GITEA_RESPONSE_TOO_LARGE;写入响应无法完整确认仍为 REMOTE_RESULT_UNKNOWN,持久化阻断后续仓库写入,不自动重试。 + +三个端的 `files?download=true` 改为同步流式转发:使用 Gitea 1.26.4 的 `contents-ext`(不传 `includes`)只查元数据,元数据响应上限 64 KiB,再按同一 SHA 请求 `/raw/`,以 64 KiB 缓冲区写出。单文件下载独立使用上述 1 GiB / 5m 配置;`FILE_MAX_CONCURRENT` 名额持有到传输完成、失败或取消。元数据、Content-Length 和实际读取字节数均校验,大小超限为 `413 FILE_TOO_LARGE`,超时为 `504 FILE_DOWNLOAD_TIMEOUT`。JSON 预览、编辑、上传和归档配额不变。 + +部署单文件下载路径时关闭代理响应缓冲和缓存,代理读取/发送超时默认至少预留 **400 秒**,并与调整后的下载预算保持一致;前端请求超时也至少预留 400 秒。应用下载预算约束上游处理和读取,慢客户端写出还受代理及容器超时约束。发包后故障直接停止传输,HTTP 可能仍为 200;前端应丢弃取消或截断的文件,不自动重试。上线后使用原 `doc/mujoco.gif` 下载链接验证文件长度与内容摘要。 ### 依赖安全核对(2026-10-22) diff --git a/docs/frontend-repository-download-git.md b/docs/frontend-repository-download-git.md index 268af6d..e93e913 100644 --- a/docs/frontend-repository-download-git.md +++ b/docs/frontend-repository-download-git.md @@ -114,7 +114,7 @@ GET /api/community/projects/7m4k9x2p8q6r/archive?format=tar.gz 页面提供“下载 ZIP”“下载 TAR.GZ”,历史版本页面的工作区下载传入对应完整 SHA。点击后展示下载中状态、防止重复点击,提供取消;成功按 Blob 保存,失败先按 JSON 解析。工作区不能用无法附加 Bearer 请求头的普通下载链接,也不能把 Token 拼到 URL。 -归档默认最大 1 GiB、上游处理和读取预算 5 分钟、每实例并发 2、每 IP 每分钟 10 次,部署可调整。此限制独立于普通文件 50 MiB 读取限制。通用请求拦截器不要使用 10~30 秒超时提前截断归档;如设置超时,应与后端约定,默认预留至少 400 秒。 +归档默认最大 1 GiB、上游处理和读取预算 5 分钟、每实例并发 2、每 IP 每分钟 10 次,部署可调整。此限制独立于 JSON 文件预览的 50 MiB 限制及单文件流式下载配置。通用请求拦截器不要使用 10~30 秒超时提前截断归档;如设置超时,应与后端约定,默认预留至少 400 秒。 发附件前的错误返回 JSON;传输开始后的错误会停止响应,此时 HTTP 可能仍为 200,不能再返回 JSON。网络错误、超时、取消后应丢弃残缺文件;没有长度信息时浏览器也不一定识别所有截断情况,解压失败应提示手动重新下载。不提供断点续传、下载任务或进度查询;没有总长度时不要展示虚构百分比。 diff --git a/docs/vibemake-api.md b/docs/vibemake-api.md index 282de97..68cf81c 100644 --- a/docs/vibemake-api.md +++ b/docs/vibemake-api.md @@ -824,9 +824,13 @@ entries 仅包含当前目录的直接子项,目录在前,名称升序。展 } ``` -只有 UTF-8、没有 NUL 字节且不超过文本编辑限制的文件返回 editable=true。二进制文件或超过编辑限制的文本返回 base64,不能直接把 base64 放进编辑器提交;使用上传替换。文件和下载均受单文件读取上限限制。 +只有 UTF-8、没有 NUL 字节且不超过文本编辑限制的文件返回 editable=true。二进制文件或超过编辑限制的文本返回 base64,不能直接把 base64 放进编辑器提交;使用上传替换。JSON 文件预览默认上限 50 MiB,在线文本编辑默认上限 1 MiB。 -`download=true` 成功返回 `application/octet-stream`,带 `Content-Disposition: attachment`,不套 JSON。工作区下载需要带 Bearer Token,应使用 fetch 获取 Blob,直接使用 a 标签或 window.open 无法附加该请求头。失败时按 JSON 错误响应处理。 +`download=true` 使用独立的流式下载链路,默认最大 **1 GiB**(`FILE_DOWNLOAD_MAX_BYTES=1073741824`),不受 JSON 预览的 50 MiB 上限影响。成功返回 `application/octet-stream`,带 `Content-Disposition: attachment`、UTF-8 文件名、`Content-Length`、`Cache-Control: no-store` 和 `X-Request-Id`,不套 JSON。下载受理时固定 SHA;工作区省略版本时使用当时的 main,社区使用当前发布版本。空文件可正常下载,LFS 指针按仓库原文返回,不自动展开实体。 + +工作区下载需要带 Bearer Token,应使用 fetch 获取 Blob,直接使用 a 标签或 window.open 无法附加该请求头。上游处理和读取总预算默认 **5 分钟**(`FILE_DOWNLOAD_TIMEOUT=5m`),客户端超时建议至少预留 **400 秒**。沿用文件读取限流及 `FILE_MAX_CONCURRENT` 名额,名额持有到传输结束。大小超限为 `413 FILE_TOO_LARGE`,超时为 `504 FILE_DOWNLOAD_TIMEOUT`,并发不足为 `429 FILE_BUSY`。 + +发包前失败返回标准 JSON;传输开始后失败会停止响应,HTTP 状态可能仍为 200,此时不再返回 JSON。网络错误、取消或长度不符应丢弃残缺文件,由用户手动重试;不要自动重试。不提供 Range 断点续传。 ### 社区目录和文件 @@ -859,7 +863,7 @@ entries 仅包含当前目录的直接子项,目录在前,名称升序。展 **这是 Git 仓库归档,不是项目全部资源打包。** LFS 文件保留指针,不替换为真实大文件;子模块依赖不递归下载。单独绑定于 OSS 的封面、URDF、STEP、GLB、零件、拼装及行走资源不会加入归档;如果仓库本身包含同类文件,则按仓库内容正常导出。 -归档使用独立配置,不受普通单文件读取 50 MiB、仓库上传 200 MiB 限制影响: +归档使用独立配置,不受 JSON 单文件预览 50 MiB、单文件流式下载或仓库上传 200 MiB 限制影响: | 配置 | 默认值 | 含义 | | --- | --- | --- | @@ -1007,7 +1011,8 @@ Blob 下载会占用浏览器存储或内存,不代表服务端将整个归档 | 限制 | 数值 | | --- | --- | | 在线创建或编辑的单个文本内容 | 1 MiB,1048576 字节 | -| 上传或读取的单文件 | 50 MiB,52428800 字节 | +| 上传或 JSON 预览的单文件 | 50 MiB,52428800 字节 | +| 流式下载的单文件 | 1 GiB,1073741824 字节,`FILE_DOWNLOAD_MAX_BYTES` 可配置 | | 本次上传文件或提交内容总量 | 200 MiB,209715200 字节 | | multipart 整个请求 | 202 MiB,预留 2 MiB 表单开销 | | 单批变更或上传文件数量 | 500 | @@ -1088,7 +1093,7 @@ commitSha 必须为当前 main 的完整 SHA,不能发布历史提交。成功 Fork 使用 Git 仓库级快照复制:冻结校验通过的 publishedCommitSha,保留该提交完整的文件树,在新私有仓库 main 上创建一个无父提交的初始版本,不继承源仓库的历史、其他分支或标签,不自动同步上游。允许同一用户及多个用户多次 Fork 同一来源。新副本的提交 SHA 与 sourceCommitSha 不同,应通过工作区 tree 接口获取新版本。 -复制保留普通文件、执行权限、符号链接、`.gitmodules` 和子模块引用;不递归下载子模块依赖。Git LFS 指针及发布快照引用的实体会完整复制,并从副本独立下载校验。缺失的 LFS 实体会导致 Fork 失败。复制不使用普通文件接口的 50 MiB 读取上限,在线文件浏览、下载及 LFS 展开能力仍受原限制。URDF 按受理时绑定的当前清单独立复制到新目录,复制时校验 ETag,封面继承原 URL;Git、LFS 和 OSS 全部完成后才返回 READY。 +复制保留普通文件、执行权限、符号链接、`.gitmodules` 和子模块引用;不递归下载子模块依赖。Git LFS 指针及发布快照引用的实体会完整复制,并从副本独立下载校验。缺失的 LFS 实体会导致 Fork 失败。复制不使用 JSON 文件预览的 50 MiB 上限或流式下载的 1 GiB 上限;JSON 预览与下载各自受配置限制,在线读取不展开 LFS 实体。URDF 按受理时绑定的当前清单独立复制到新目录,复制时校验 ETag,封面继承原 URL;Git、LFS 和 OSS 全部完成后才返回 READY。 后端部署需要 Git 和 git-lfs。默认 Fork 总预算5分钟、单实例最多同时2次,每次临时磁盘上限5 GiB(含 Git 对象、LFS 源缓存和副本校验缓存)。配置分别为 FORK_TIMEOUT、FORK_MAX_CONCURRENT、FORK_MAX_TEMPORARY_BYTES;临时目录可通过 FORK_TEMPORARY_DIRECTORY 指定。依赖不可用或并发超限时,在创建项目记录前拒绝。普通创建和其他 API 不依赖 Fork 运行环境。 @@ -1406,7 +1411,9 @@ if (result.status !== 'CHAT') { | 409 | CONFLICT | 工作区或发布版本变化,或仓库未就绪,按 message 刷新版本或核对状态 | | 409 | GITEA_CONFLICT | 仓库内容冲突,刷新目录和文件后由用户处理 | | 409 | REMOTE_RESULT_UNKNOWN | 上一次仓库写入结果待核对,暂不能继续该项目的仓库写入 | -| 413 | FILE_TOO_LARGE | 文件读取、编辑或上传内容超限,提示大小限制 | +| 413 | FILE_TOO_LARGE | JSON 预览、流式下载、编辑或上传内容超过各自大小上限,展示 message 中的限制 | +| 499 | FILE_DOWNLOAD_CANCELLED | 下载取消,丢弃残缺文件 | +| 504 | FILE_DOWNLOAD_TIMEOUT | 下载超时,丢弃残缺文件,可手动重试 | | 413 | UPLOAD_TOO_LARGE | 整体 multipart 请求超限 | | 413 | UPLOAD_TOO_MANY_PARTS | multipart 文件或表单项数量超限,减少文件或重复字段 | | 415 | UNSUPPORTED_MEDIA_TYPE | 检查 Content-Type,JSON 与 FormData 不要混用 | @@ -1740,4 +1747,4 @@ async function createGitToken(projectCode: string, token: string) { | GIT_TRANSFER_TOO_LARGE、GIT_METADATA_TOO_LARGE、LFS_TOO_LARGE / 413 | 缩小请求或联系运维核对配置 | | GIT_TIMEOUT / 504、GIT_UPSTREAM_ERROR / 502、REMOTE_RESULT_UNKNOWN / 409或503 | 涉及 push 时先刷新工作区及历史,结果未确认前不能重复写入 | -默认 pack 5 GiB、LFS 单对象1 GiB、元数据8 MiB、batch最多500个对象;每次 Git/LFS 请求预算10分钟,含请求及响应流。读取/push/LFS 每实例并发4/2/8;每用户每分钟 Git120次、LFS600次,每IP每分钟最多20次认证失败。均可配置,独立于 JSON、multipart、在线50 MiB文件读取和归档下载限制。中途超限、超时或断开会终止传输;不提供任务、进度查询或断点续传接口。 +默认 pack 5 GiB、LFS 单对象1 GiB、元数据8 MiB、batch最多500个对象;每次 Git/LFS 请求预算10分钟,含请求及响应流。读取/push/LFS 每实例并发4/2/8;每用户每分钟 Git120次、LFS600次,每IP每分钟最多20次认证失败。均可配置,独立于 JSON、multipart、50 MiB 文件预览、单文件流式下载和归档下载限制。中途超限、超时或断开会终止传输;不提供任务、进度查询或断点续传接口。 diff --git a/vibemake-adminapi/src/main/java/com/linkerbot/vibemake/adminapi/controller/AdminProjectController.java b/vibemake-adminapi/src/main/java/com/linkerbot/vibemake/adminapi/controller/AdminProjectController.java index 6f0d7de..2615d25 100644 --- a/vibemake-adminapi/src/main/java/com/linkerbot/vibemake/adminapi/controller/AdminProjectController.java +++ b/vibemake-adminapi/src/main/java/com/linkerbot/vibemake/adminapi/controller/AdminProjectController.java @@ -5,9 +5,11 @@ import com.linkerbot.vibemake.bean.Contracts.*; import com.linkerbot.vibemake.common.*; import com.linkerbot.vibemake.service.*; import com.linkerbot.vibemake.service.admin.AdminManagementService; -import com.linkerbot.vibemake.service.gitea.GiteaClient; +import com.linkerbot.vibemake.service.download.FileDownloadService; +import com.linkerbot.vibemake.service.download.FileDownloadResponse; import jakarta.validation.Valid; +import jakarta.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; @@ -15,7 +17,6 @@ import org.springframework.http.*; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; -import java.nio.charset.StandardCharsets; import java.util.List; @RestController @@ -25,6 +26,7 @@ public class AdminProjectController { private final AdminManagementService management; private final ProjectService projects; private final FileService files; + private final FileDownloadService downloads; private final UrdfService urdf; @GetMapping @@ -91,22 +93,13 @@ public class AdminProjectController { @PathVariable String projectCode, @RequestParam String path, @RequestParam(required = false) String commitSha, - @RequestParam(defaultValue = "false") boolean download) { + @RequestParam(defaultValue = "false") boolean download, + HttpServletResponse response) { if (!download) return ResponseEntity.ok( ApiResponse.success(files.ownerFileView(0, projectCode, path, commitSha))); - GiteaClient.StoredFile file = files.ownerFile(0, projectCode, path, commitSha); - return ResponseEntity.ok() - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .header( - HttpHeaders.CONTENT_DISPOSITION, - ContentDisposition.attachment() - .filename( - file.path().substring(file.path().lastIndexOf('/') + 1), - StandardCharsets.UTF_8) - .build() - .toString()) - .body(file.bytes()); + FileDownloadResponse.write(response, destination -> downloads.owner(0, projectCode, path, commitSha, destination)); + return null; } @GetMapping("/{projectCode}/commits") diff --git a/vibemake-adminapi/src/main/resources/application.yml b/vibemake-adminapi/src/main/resources/application.yml index c85c365..ee84d06 100644 --- a/vibemake-adminapi/src/main/resources/application.yml +++ b/vibemake-adminapi/src/main/resources/application.yml @@ -108,6 +108,9 @@ vibemake: max-text-bytes: 1048576 max-file-bytes: 52428800 max-request-bytes: 209715200 + file-download: + max-bytes: ${FILE_DOWNLOAD_MAX_BYTES:1073741824} + timeout: ${FILE_DOWNLOAD_TIMEOUT:5m} gitea: connect-timeout: 5s request-timeout: 30s diff --git a/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/AdminWorkflowTest.java b/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/AdminWorkflowTest.java index fb819c9..fa0c1d7 100644 --- a/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/AdminWorkflowTest.java +++ b/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/AdminWorkflowTest.java @@ -108,6 +108,7 @@ class AdminWorkflowTest { @Autowired FakeGiteaClient gitea; @Autowired ProjectService projectService; @Autowired FileService fileService; + @Autowired com.linkerbot.vibemake.service.config.FileDownloadProperties fileDownloadProperties; @Autowired UserService userService; @Autowired ProjectLocks locks; @Autowired com.linkerbot.vibemake.service.write.RepositoryWrites writes; @@ -189,6 +190,44 @@ class AdminWorkflowTest { .asText(); } + @Test + void fileDownloadsUseStreamingWithCurrentAndHistoricalVersionsAndOwnLimit() throws Exception { + String code = createProject(); + String endpoint = "/admin-api/projects/" + code; + String first = initialSha(code); + request(signed(body(post(endpoint + "/commits"), Map.of( + "expectedCommitSha", first, "message", "更新下载内容", "changes", List.of( + Map.of("action", "UPDATE_FILE", "path", "README.md", "content", "后台版本"))))), 200); + for (boolean history : List.of(false, true)) { + String content = history ? "initial" : "后台版本"; + MockHttpServletRequestBuilder download = signed(get(endpoint + "/files") + .param("path", "README.md").param("download", "true")); + if (history) download.param("commitSha", first); + MvcResult result = mvc.perform(download).andReturn(); + assertThat(result.getResponse().getStatus()).isEqualTo(200); + assertThat(result.getResponse().getContentType()).isEqualTo("application/octet-stream"); + assertThat(result.getResponse().getContentAsByteArray()).isEqualTo(content.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + assertThat(result.getResponse().getHeader("Content-Disposition")).contains("attachment"); + assertThat(result.getResponse().getHeader("Content-Length")) + .isEqualTo(Integer.toString(content.getBytes(java.nio.charset.StandardCharsets.UTF_8).length)); + assertThat(result.getResponse().getHeader("Cache-Control")).isEqualTo("no-store"); + assertThat(result.getResponse().getHeader("X-Request-Id")).isNotBlank(); + } + assertThat(request(signed(get(endpoint + "/files").param("path", "README.md")), 200) + .path("data").path("content").asText()).isEqualTo("后台版本"); + request(get(endpoint + "/files").param("path", "README.md").param("download", "true"), 401); + long maximum = fileDownloadProperties.getMaxBytes(); + try { + fileDownloadProperties.setMaxBytes(1); + MvcResult result = mvc.perform(signed(get(endpoint + "/files") + .param("path", "README.md").param("download", "true"))).andReturn(); + assertThat(result.getResponse().getStatus()).isEqualTo(413); + assertThat(json.readTree(result.getResponse().getContentAsByteArray()).path("code").asText()).isEqualTo("FILE_TOO_LARGE"); + assertThat(result.getResponse().getHeader("Content-Disposition")).isNull(); + assertThat(result.getResponse().getHeader("Content-Length")).isNull(); + } finally { fileDownloadProperties.setMaxBytes(maximum); } + } + @Test void publicationTimeCanBeSetOmittedClearedAndIsProtectedFromOrdinaryUsers() throws Exception { String code = createProject(); diff --git a/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/FakeGiteaClient.java b/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/FakeGiteaClient.java index 45b0763..e865281 100644 --- a/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/FakeGiteaClient.java +++ b/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/FakeGiteaClient.java @@ -5,6 +5,8 @@ import com.linkerbot.vibemake.common.PageResponse; import com.linkerbot.vibemake.service.gitea.*; import java.security.MessageDigest; +import java.io.ByteArrayInputStream; +import java.time.Duration; import java.util.*; import java.util.concurrent.atomic.AtomicLong; @@ -164,6 +166,16 @@ final class FakeGiteaClient implements GiteaClient { return new StoredFile(path, blob(bytes), bytes.clone()); } + @Override + public synchronized FileDownload openFile(Repository repository, String sha, String path, + Duration timeout, long maxBytes) { + Map snapshot = repo(repository).snapshots.get(sha); + byte[] bytes = snapshot == null ? null : snapshot.get(path); + if (bytes == null) throw new GiteaFailure(404, "NOT_FOUND", "文件不存在", false); + if (bytes.length > maxBytes) throw new GiteaFailure(413, "FILE_TOO_LARGE", "文件超过下载限制", false); + return new FileDownload(new ByteArrayInputStream(bytes), bytes.length); + } + @Override public synchronized CommitView commit( Repository repository, diff --git a/vibemake-api/src/main/java/com/linkerbot/vibemake/api/controller/VibemakeController.java b/vibemake-api/src/main/java/com/linkerbot/vibemake/api/controller/VibemakeController.java index 718061b..1dd0a93 100644 --- a/vibemake-api/src/main/java/com/linkerbot/vibemake/api/controller/VibemakeController.java +++ b/vibemake-api/src/main/java/com/linkerbot/vibemake/api/controller/VibemakeController.java @@ -4,12 +4,15 @@ import com.linkerbot.vibemake.bean.Contracts.*; import com.linkerbot.vibemake.common.*; import com.linkerbot.vibemake.security.CurrentUser; import com.linkerbot.vibemake.service.*; +import com.linkerbot.vibemake.service.download.FileDownloadService; +import com.linkerbot.vibemake.service.download.FileDownloadResponse; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.security.SecurityRequirements; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; +import jakarta.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; @@ -24,6 +27,7 @@ import java.util.List; public class VibemakeController { private final ProjectService projects; private final FileService files; + private final FileDownloadService downloads; private final InteractionService interactions; @GetMapping("/api/project-types") @@ -71,12 +75,13 @@ public class VibemakeController { @PathVariable String projectCode, @RequestParam String path, @RequestParam(defaultValue = "false") boolean download, - @RequestParam(required = false) String commitSha) { + @RequestParam(required = false) String commitSha, + HttpServletResponse response) { rejectRef(commitSha); - return download - ? WorkspaceController.download(files.vibemakeFile(projectCode, path)) - : ResponseEntity.ok( - ApiResponse.success(files.vibemakeFileView(projectCode, path))); + if (!download) + return ResponseEntity.ok(ApiResponse.success(files.vibemakeFileView(projectCode, path))); + FileDownloadResponse.write(response, destination -> downloads.community(projectCode, path, destination)); + return null; } private void rejectRef(String sha) { diff --git a/vibemake-api/src/main/java/com/linkerbot/vibemake/api/controller/WorkspaceController.java b/vibemake-api/src/main/java/com/linkerbot/vibemake/api/controller/WorkspaceController.java index 7c1d0a3..64c7c13 100644 --- a/vibemake-api/src/main/java/com/linkerbot/vibemake/api/controller/WorkspaceController.java +++ b/vibemake-api/src/main/java/com/linkerbot/vibemake/api/controller/WorkspaceController.java @@ -4,12 +4,14 @@ import com.linkerbot.vibemake.bean.Contracts.*; import com.linkerbot.vibemake.common.*; import com.linkerbot.vibemake.security.CurrentUser; import com.linkerbot.vibemake.service.*; -import com.linkerbot.vibemake.service.gitea.GiteaClient; +import com.linkerbot.vibemake.service.download.FileDownloadService; +import com.linkerbot.vibemake.service.download.FileDownloadResponse; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; +import jakarta.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; @@ -17,7 +19,6 @@ import org.springframework.http.*; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; -import java.nio.charset.StandardCharsets; import java.util.List; @RestController @@ -27,6 +28,7 @@ import java.util.List; public class WorkspaceController { private final ProjectService projects; private final FileService files; + private final FileDownloadService downloads; @GetMapping @Operation(summary = "我的项目列表") @@ -82,25 +84,14 @@ public class WorkspaceController { @PathVariable String projectCode, @RequestParam String path, @RequestParam(required = false) String commitSha, - @RequestParam(defaultValue = "false") boolean download) { + @RequestParam(defaultValue = "false") boolean download, + HttpServletResponse response) { long user = CurrentUser.required(); if (!download) return ResponseEntity.ok( ApiResponse.success(files.ownerFileView(user, projectCode, path, commitSha))); - return download(files.ownerFile(user, projectCode, path, commitSha)); - } - - public static ResponseEntity download(GiteaClient.StoredFile file) { - String name = file.path().substring(file.path().lastIndexOf('/') + 1); - return ResponseEntity.ok() - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .header( - HttpHeaders.CONTENT_DISPOSITION, - ContentDisposition.attachment() - .filename(name, StandardCharsets.UTF_8) - .build() - .toString()) - .body(file.bytes()); + FileDownloadResponse.write(response, destination -> downloads.owner(user, projectCode, path, commitSha, destination)); + return null; } @PostMapping("/{projectCode}/commits") diff --git a/vibemake-api/src/main/resources/application.yml b/vibemake-api/src/main/resources/application.yml index 11c069a..cb7bb73 100644 --- a/vibemake-api/src/main/resources/application.yml +++ b/vibemake-api/src/main/resources/application.yml @@ -127,6 +127,9 @@ vibemake: max-text-bytes: 1048576 max-file-bytes: 52428800 max-request-bytes: 209715200 + file-download: + max-bytes: ${FILE_DOWNLOAD_MAX_BYTES:1073741824} + timeout: ${FILE_DOWNLOAD_TIMEOUT:5m} gitea: connect-timeout: 5s request-timeout: 30s diff --git a/vibemake-api/src/test/java/com/linkerbot/vibemake/api/FakeGiteaClient.java b/vibemake-api/src/test/java/com/linkerbot/vibemake/api/FakeGiteaClient.java index cb024cb..78764ce 100644 --- a/vibemake-api/src/test/java/com/linkerbot/vibemake/api/FakeGiteaClient.java +++ b/vibemake-api/src/test/java/com/linkerbot/vibemake/api/FakeGiteaClient.java @@ -145,6 +145,16 @@ final class FakeGiteaClient implements GiteaClient { return new StoredFile(path, blob(bytes), bytes.clone()); } + @Override + public synchronized FileDownload openFile(Repository repository, String sha, String path, + Duration timeout, long maxBytes) { + Map snapshot = repo(repository).snapshots.get(sha); + byte[] bytes = snapshot == null ? null : snapshot.get(path); + if (bytes == null) throw new GiteaFailure(404, "NOT_FOUND", "文件不存在", false); + if (bytes.length > maxBytes) throw new GiteaFailure(413, "FILE_TOO_LARGE", "文件超过下载限制", false); + return new FileDownload(new ByteArrayInputStream(bytes), bytes.length); + } + @Override public synchronized CommitView commit( Repository repository, diff --git a/vibemake-api/src/test/java/com/linkerbot/vibemake/api/FileDownloadApiTest.java b/vibemake-api/src/test/java/com/linkerbot/vibemake/api/FileDownloadApiTest.java new file mode 100644 index 0000000..255fda5 --- /dev/null +++ b/vibemake-api/src/test/java/com/linkerbot/vibemake/api/FileDownloadApiTest.java @@ -0,0 +1,244 @@ +package com.linkerbot.vibemake.api; + +import static org.assertj.core.api.Assertions.*; +import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkerbot.vibemake.common.BusinessException; +import com.linkerbot.vibemake.service.config.FileDownloadProperties; +import com.linkerbot.vibemake.service.gitea.GiteaClient; +import com.linkerbot.vibemake.service.gitea.GiteaFailure; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.autoconfigure.web.servlet.MockMvcPrint; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.http.MediaType; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +@SpringBootTest(properties = { + "spring.datasource.url=jdbc:h2:mem:filedownloads;MODE=MySQL;DB_CLOSE_DELAY=-1", + "spring.datasource.username=sa", "spring.datasource.password=", + "spring.datasource.driver-class-name=org.h2.Driver", + "vibemake.ai.enabled=false", "vibemake.oss.enabled=false", + "vibemake.limits.enabled=false", "vibemake.limits.file-concurrent=1", + "vibemake.repository-write.worker-enabled=false", "logging.level.root=WARN"}) +@ActiveProfiles("test") +@AutoConfigureMockMvc(print = MockMvcPrint.NONE) +@Import(WorkflowTest.Config.class) +@Timeout(20) +class FileDownloadApiTest { + @Autowired MockMvc mvc; + @Autowired ObjectMapper json; + @Autowired JdbcTemplate jdbc; + @Autowired FileDownloadProperties properties; + @MockitoSpyBean FakeGiteaClient gitea; + + @BeforeEach + void resetState() throws Exception { + reset(gitea); + gitea.reset(); + properties.setMaxBytes(1024L * 1024 * 1024); + SchemaSupport.reset(jdbc); + } + + private JsonNode call(MockHttpServletRequestBuilder request) throws Exception { + return json.readTree(mvc.perform(request).andExpect(status().isOk()).andReturn() + .getResponse().getContentAsByteArray()).path("data"); + } + + private MockHttpServletRequestBuilder body(MockHttpServletRequestBuilder request, Object input) throws Exception { + return request.contentType(MediaType.APPLICATION_JSON).content(json.writeValueAsBytes(input)); + } + + private MockHttpServletRequestBuilder auth(MockHttpServletRequestBuilder request, String token) { + return request.header("Authorization", "Bearer " + token); + } + + private String login(String email) throws Exception { + String captcha = call(get("/api/auth/captcha")).path("captchaCode").asText(); + return call(body(post("/api/auth/login"), Map.of("email", email, + "captchaCode", captcha, "captchaAnswer", "ABCDE"))).path("token").asText(); + } + + private String create(String token) throws Exception { + return call(body(auth(post("/api/workspace/projects"), token), + Map.of("name", "文件下载测试", "projectType", "GENERAL"))).path("projectCode").asText(); + } + + private String head(String token, String code) throws Exception { + return call(auth(get("/api/workspace/projects/" + code + "/tree"), token)).path("commitSha").asText(); + } + + private String commit(String token, String code, String sha, String action, String path, String content) throws Exception { + return call(body(auth(post("/api/workspace/projects/" + code + "/commits"), token), + Map.of("expectedCommitSha", sha, "message", "更新文件", "changes", List.of( + Map.of("action", action, "path", path, "content", content))))) + .path("commitSha").asText(); + } + + @Test + void downloadsFrozenPublishedCurrentAndHistoricalFilesWithChineseNames() throws Exception { + String token = login("file-versions@example.invalid"); + String code = create(token); + String path = "doc/动画 空格.gif"; + String first = commit(token, code, head(token, code), "CREATE_FILE", path, "发布版本"); + call(body(auth(put("/api/workspace/projects/" + code + "/publication"), token), Map.of("commitSha", first))); + commit(token, code, first, "UPDATE_FILE", path, "工作区版本"); + for (String context : List.of("community", "workspace", "history")) { + String content = context.equals("workspace") ? "工作区版本" : "发布版本"; + MockHttpServletRequestBuilder request = get("/api/" + (context.equals("community") ? "community" : "workspace") + + "/projects/" + code + "/files").param("path", path).param("download", "true") + .header("Origin", "https://dev-vibemake.robotquan.com"); + if (!context.equals("community")) auth(request, token); + if (context.equals("history")) request.param("commitSha", first); + mvc.perform(request).andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_OCTET_STREAM)) + .andExpect(content().bytes(content.getBytes(StandardCharsets.UTF_8))) + .andExpect(header().string("Content-Length", Integer.toString(content.getBytes(StandardCharsets.UTF_8).length))) + .andExpect(header().string("Content-Disposition", containsString("filename*="))) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(header().exists("X-Request-Id")) + .andExpect(header().string("Access-Control-Expose-Headers", containsString("Content-Disposition"))); + } + } + + @Test + void permissionsReadinessAndCommunityRefRestrictionsRemainEnforced() throws Exception { + String token = login("file-owner@example.invalid"); + String other = login("file-other@example.invalid"); + String code = create(token); + String workspace = "/api/workspace/projects/" + code + "/files"; + String community = "/api/community/projects/" + code + "/files"; + mvc.perform(get(workspace).param("path", "README.md").param("download", "true")).andExpect(status().isUnauthorized()); + mvc.perform(auth(get(workspace).param("path", "README.md").param("download", "true"), other)).andExpect(status().isNotFound()); + mvc.perform(get(community).param("path", "README.md").param("download", "true")).andExpect(status().isNotFound()); + call(body(auth(put("/api/workspace/projects/" + code + "/publication"), token), Map.of("commitSha", head(token, code)))); + for (String ref : List.of("", "a".repeat(40))) + mvc.perform(get(community).param("path", "README.md").param("download", "true").param("commitSha", ref)) + .andExpect(status().isBadRequest()); + mvc.perform(auth(get(workspace).param("path", "../README.md").param("download", "true"), token)).andExpect(status().isBadRequest()); + jdbc.update("UPDATE project SET repo_status='CREATING' WHERE project_code=?", code); + mvc.perform(auth(get(workspace).param("path", "README.md").param("download", "true"), token)).andExpect(status().isConflict()); + jdbc.update("UPDATE project SET repo_status='READY' WHERE project_code=?", code); + call(auth(delete("/api/workspace/projects/" + code + "/publication"), token)); + mvc.perform(get(community).param("path", "README.md").param("download", "true")).andExpect(status().isNotFound()); + call(auth(delete("/api/workspace/projects/" + code), token)); + mvc.perform(auth(get(workspace).param("path", "README.md").param("download", "true"), token)).andExpect(status().isNotFound()); + } + + @Test + void downloadBypassesJsonReaderAndAppliesItsOwnLimit() throws Exception { + String token = login("file-limits@example.invalid"); + String code = create(token); + String endpoint = "/api/workspace/projects/" + code + "/files"; + doThrow(new BusinessException(413, "FILE_TOO_LARGE", "文件超过读取限制")) + .when(gitea).file(any(), anyString(), eq("README.md")); + mvc.perform(auth(get(endpoint).param("path", "README.md"), token)).andExpect(status().isPayloadTooLarge()); + mvc.perform(auth(get(endpoint).param("path", "README.md").param("download", "true"), token)) + .andExpect(status().isOk()).andExpect(content().string("initial")); + verify(gitea, times(1)).file(any(), anyString(), anyString()); + properties.setMaxBytes(1); + mvc.perform(auth(get(endpoint).param("path", "README.md").param("download", "true"), token)) + .andExpect(status().isPayloadTooLarge()).andExpect(jsonPath("$.code").value("FILE_TOO_LARGE")) + .andExpect(header().doesNotExist("Content-Disposition")).andExpect(header().doesNotExist("Content-Length")); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void errorsCloseUpstreamAndNeverAppendJsonToCommittedFile(boolean commit) throws Exception { + String token = login("file-failure@example.invalid"); + String code = create(token); + AtomicBoolean closed = new AtomicBoolean(); + InputStream failing = new InputStream() { + private boolean sent; + @Override public int read() { throw new UnsupportedOperationException(); } + @Override public int read(byte[] bytes, int offset, int length) { + if (commit && !sent) { + sent = true; + int size = Math.min(length, 16384); + java.util.Arrays.fill(bytes, offset, offset + size, (byte) 'x'); + return size; + } + throw new GiteaFailure(504, "FILE_DOWNLOAD_TIMEOUT", "timeout", false); + } + @Override public void close() { closed.set(true); } + }; + doReturn(new GiteaClient.FileDownload(failing, 100000)).when(gitea) + .openFile(any(), anyString(), anyString(), any(), anyLong()); + MvcResult result = mvc.perform(auth(get("/api/workspace/projects/" + code + "/files") + .param("path", "README.md").param("download", "true"), token)).andReturn(); + assertThat(closed.get()).isTrue(); + if (commit) { + assertThat(result.getResponse().getStatus()).isEqualTo(200); + assertThat(result.getResponse().getContentAsString()).isEqualTo("x".repeat(16384)); + } else { + assertThat(result.getResponse().getStatus()).isEqualTo(504); + assertThat(result.getResponse().getHeader("Content-Disposition")).isNull(); + assertThat(result.getResponse().getHeader("Content-Length")).isNull(); + assertThat(json.readTree(result.getResponse().getContentAsByteArray()).path("code").asText()).isEqualTo("FILE_DOWNLOAD_TIMEOUT"); + } + } + + @Test + void concurrentPermitIsHeldDuringStreamingAndReleasedAfterFailure() throws Exception { + String token = login("file-concurrency@example.invalid"); + String code = create(token); + String endpoint = "/api/workspace/projects/" + code + "/files"; + CountDownLatch reading = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicBoolean closed = new AtomicBoolean(); + InputStream blocking = new InputStream() { + @Override public int read() { throw new UnsupportedOperationException(); } + @Override public int read(byte[] bytes, int offset, int length) { + reading.countDown(); + try { if (!release.await(5, TimeUnit.SECONDS)) throw new AssertionError("下载未释放"); } + catch (InterruptedException error) { throw new AssertionError(error); } + throw new GiteaFailure(502, "GITEA_ERROR", "fixture", false); + } + @Override public void close() { closed.set(true); } + }; + doReturn(new GiteaClient.FileDownload(blocking, 10)).when(gitea) + .openFile(any(), anyString(), anyString(), any(), anyLong()); + try (ExecutorService worker = Executors.newVirtualThreadPerTaskExecutor()) { + Future first = worker.submit(() -> mvc.perform(auth(get(endpoint) + .param("path", "README.md").param("download", "true"), token)).andReturn()); + try { + assertThat(reading.await(5, TimeUnit.SECONDS)).isTrue(); + mvc.perform(auth(get(endpoint).param("path", "README.md").param("download", "true"), token)) + .andExpect(status().isTooManyRequests()).andExpect(jsonPath("$.code").value("FILE_BUSY")) + .andExpect(header().exists("Retry-After")); + } finally { release.countDown(); } + assertThat(first.get(5, TimeUnit.SECONDS).getResponse().getStatus()).isEqualTo(502); + } + assertThat(closed.get()).isTrue(); + doCallRealMethod().when(gitea).openFile(any(), anyString(), anyString(), any(), anyLong()); + mvc.perform(auth(get(endpoint).param("path", "README.md").param("download", "true"), token)).andExpect(status().isOk()); + } +} diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/config/FileDownloadProperties.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/config/FileDownloadProperties.java new file mode 100644 index 0000000..a91593b --- /dev/null +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/config/FileDownloadProperties.java @@ -0,0 +1,27 @@ +package com.linkerbot.vibemake.service.config; + +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Positive; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; +import org.springframework.validation.annotation.Validated; + +import java.time.Duration; + +@Getter +@Setter +@Validated +@Component +@ConfigurationProperties("vibemake.file-download") +public class FileDownloadProperties { + @Positive private long maxBytes = 1024L * 1024 * 1024; + private Duration timeout = Duration.ofMinutes(5); + + @AssertTrue(message = "文件下载超时必须在 100 毫秒至 1 小时之间") + public boolean isTimeoutValid() { + return timeout != null && timeout.compareTo(Duration.ofMillis(100)) >= 0 + && timeout.compareTo(Duration.ofHours(1)) <= 0; + } +} diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/download/FileDownloadResponse.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/download/FileDownloadResponse.java new file mode 100644 index 0000000..38e7851 --- /dev/null +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/download/FileDownloadResponse.java @@ -0,0 +1,57 @@ +package com.linkerbot.vibemake.service.download; + +import com.linkerbot.vibemake.common.BusinessException; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +/** 同步传输结束后才返回控制器,文件并发名额由现有过滤器持有至连接关闭。 */ +public final class FileDownloadResponse { + private FileDownloadResponse() {} + + @FunctionalInterface + public interface Transfer { + void run(FileDownloadService.Destination destination) throws IOException; + } + + public static void write(HttpServletResponse response, Transfer transfer) { + response.setHeader(HttpHeaders.CACHE_CONTROL, "no-store"); + try { + transfer.run((filename, length) -> { + response.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.attachment() + .filename(filename, StandardCharsets.UTF_8).build().toString()); + response.setContentLengthLong(length); + return response.getOutputStream(); + }); + } catch (IOException error) { + if (!response.isCommitted()) { + resetAttachment(response); + throw new BusinessException(499, "FILE_DOWNLOAD_CANCELLED", "文件下载已取消"); + } + closeResponse(response); + } catch (RuntimeException error) { + if (!response.isCommitted()) { + resetAttachment(response); + throw error; + } + // 发包后只停止传输,禁止异常处理器向文件尾部追加 JSON。 + closeResponse(response); + } + } + + private static void resetAttachment(HttpServletResponse response) { + response.resetBuffer(); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, null); + response.setHeader(HttpHeaders.CONTENT_LENGTH, null); + response.setContentType(null); + } + + private static void closeResponse(HttpServletResponse response) { + try { response.getOutputStream().close(); } catch (IOException ignored) { } + } +} diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/download/FileDownloadService.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/download/FileDownloadService.java new file mode 100644 index 0000000..d121928 --- /dev/null +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/download/FileDownloadService.java @@ -0,0 +1,95 @@ +package com.linkerbot.vibemake.service.download; + +import com.linkerbot.vibemake.bean.entity.Project; +import com.linkerbot.vibemake.common.BusinessException; +import com.linkerbot.vibemake.common.FilePaths; +import com.linkerbot.vibemake.common.SafeExceptionLog; +import com.linkerbot.vibemake.service.ProjectService; +import com.linkerbot.vibemake.service.config.FileDownloadProperties; +import com.linkerbot.vibemake.service.gitea.GiteaClient; +import com.linkerbot.vibemake.service.gitea.GiteaFailure; +import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.io.OutputStream; +import java.time.Duration; + +@Service +@RequiredArgsConstructor +public class FileDownloadService { + private static final Logger log = LoggerFactory.getLogger(FileDownloadService.class); + private final ProjectService projects; + private final GiteaClient gitea; + private final FileDownloadProperties properties; + + @FunctionalInterface + public interface Destination { + OutputStream open(String filename, long contentLength) throws IOException; + } + + public void owner(long userId, String code, String path, String commitSha, Destination destination) + throws IOException { + Project project = projects.readyOwner(userId, code); + download(project, path, commitSha == null ? null : FilePaths.sha(commitSha), destination); + } + + public void community(String code, String path, Destination destination) throws IOException { + Project project = projects.published(code); + download(project, path, FilePaths.sha(project.getPublishedCommitSha()), destination); + } + + private void download(Project project, String path, String requestedSha, Destination destination) + throws IOException { + String normalized = FilePaths.file(path); + long started = System.nanoTime(); + long deadline = started + properties.getTimeout().toNanos(); + String code = project.getProjectCode(); + String sha = requestedSha; + long transferred = 0; + try { + GiteaClient.Repository repository = projects.repository(project); + if (sha == null) sha = FilePaths.sha(gitea.fileDownloadHead(repository, remaining(deadline))); + log.info("文件下载开始 projectCode={} sha={} path={}", code, sha, normalized); + try (GiteaClient.FileDownload file = gitea.openFile(repository, sha, normalized, + remaining(deadline), properties.getMaxBytes())) { + OutputStream output = destination.open(normalized.substring(normalized.lastIndexOf('/') + 1), + file.contentLength()); + byte[] buffer = new byte[64 * 1024]; + int size; + while ((size = file.input().read(buffer)) >= 0) { + output.write(buffer, 0, size); + transferred += size; + } + output.flush(); + } + log.info("文件下载完成 projectCode={} sha={} path={} bytes={} elapsedMs={}", + code, sha, normalized, transferred, elapsed(started)); + } catch (BusinessException error) { + log.warn("文件下载失败 projectCode={} sha={} path={} bytes={} elapsedMs={} code={}", + code, sha, normalized, transferred, elapsed(started), error.code()); + if (error.getCause() != null) SafeExceptionLog.write(log, "文件下载上游异常", error); + throw error; + } catch (IOException error) { + log.info("文件下载取消 projectCode={} sha={} path={} bytes={} elapsedMs={} code=FILE_DOWNLOAD_CANCELLED errorType={}", + code, sha, normalized, transferred, elapsed(started), error.getClass().getSimpleName()); + throw error; + } catch (RuntimeException error) { + log.warn("文件下载异常 projectCode={} sha={} path={} bytes={} elapsedMs={} code=INTERNAL_ERROR", + code, sha, normalized, transferred, elapsed(started)); + SafeExceptionLog.write(log, "文件下载异常详情", error); + throw error; + } + } + + private Duration remaining(long deadline) { + long nanos = deadline - System.nanoTime(); + if (nanos <= 0) + throw new GiteaFailure(504, "FILE_DOWNLOAD_TIMEOUT", "文件下载超时,请稍后重试", false); + return Duration.ofNanos(nanos); + } + + private long elapsed(long started) { return (System.nanoTime() - started) / 1_000_000; } +} diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/FileDownloadInputStream.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/FileDownloadInputStream.java new file mode 100644 index 0000000..9cdc8d3 --- /dev/null +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/FileDownloadInputStream.java @@ -0,0 +1,102 @@ +package com.linkerbot.vibemake.service.gitea; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** 不缓存整个文件;响应头返回后仍主动关闭超时的上游,唤醒阻塞读取。 */ +final class FileDownloadInputStream extends InputStream { + private static final ScheduledThreadPoolExecutor DEADLINES = deadlines(); + private final InputStream input; + private final long expected; + private final long maximum; + private final long deadline; + private final AtomicBoolean expired = new AtomicBoolean(); + private final ScheduledFuture timeout; + private long count; + + private static ScheduledThreadPoolExecutor deadlines() { + ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(2, task -> { + Thread thread = new Thread(task, "gitea-file-download-deadline"); + thread.setDaemon(true); + return thread; + }); + executor.setRemoveOnCancelPolicy(true); + return executor; + } + + FileDownloadInputStream(InputStream input, long expected, long maximum, long deadline) { + this.input = input; + this.expected = expected; + this.maximum = maximum; + this.deadline = deadline; + timeout = DEADLINES.schedule(() -> { + expired.set(true); + closeInput(); + }, Math.max(0, deadline - System.nanoTime()), TimeUnit.NANOSECONDS); + } + + static GiteaFailure timedOut() { + return new GiteaFailure(504, "FILE_DOWNLOAD_TIMEOUT", "文件下载超时,请稍后重试", false); + } + + static GiteaFailure tooLarge(long maximum) { + return new GiteaFailure(413, "FILE_TOO_LARGE", "文件超过下载大小限制(" + maximum + " 字节)", false); + } + + static GiteaFailure failed() { + return new GiteaFailure(502, "GITEA_ERROR", "仓库文件下载响应异常,请通过 requestId 排查", false); + } + + private void checkDeadline() { + if (expired.get() || System.nanoTime() - deadline >= 0) throw timedOut(); + } + + @Override + public int read() { + byte[] single = new byte[1]; + int size = read(single, 0, 1); + return size < 0 ? -1 : single[0] & 0xff; + } + + @Override + public int read(byte[] bytes, int offset, int length) { + Objects.checkFromIndexSize(offset, length, bytes.length); + if (length == 0) return 0; + checkDeadline(); + long remaining = Math.min(maximum, expected) - count; + int requested = remaining >= length ? length : (int) (remaining + 1); + int size; + try { + size = input.read(bytes, offset, requested); + } catch (IOException error) { + checkDeadline(); + GiteaFailure failure = failed(); + failure.initCause(error); + throw failure; + } + checkDeadline(); + if (size < 0) { + if (count != expected) throw failed(); + } else { + count += size; + if (count > maximum) throw tooLarge(maximum); + if (count > expected) throw failed(); + } + return size; + } + + private void closeInput() { + try { input.close(); } catch (IOException ignored) { /* 不输出可能含凭据的上游异常。 */ } + } + + @Override + public void close() { + timeout.cancel(false); + closeInput(); + } +} diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java index 8a375af..5b94c7b 100644 --- a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java @@ -27,6 +27,20 @@ public interface GiteaClient { record StoredFile(String path, String blobSha, byte[] bytes) {} + record FileDownload(InputStream input, long contentLength) implements AutoCloseable { + @Override + public void close() throws IOException { input.close(); } + } + + default String fileDownloadHead(Repository repository, Duration timeout) { + return head(repository); + } + + default FileDownload openFile(Repository repository, String sha, String path, + Duration timeout, long maxBytes) { + throw new UnsupportedOperationException("未实现文件流式下载"); + } + record Archive(InputStream input, long contentLength) implements AutoCloseable { @Override public void close() throws IOException { input.close(); } @@ -85,7 +99,7 @@ public interface GiteaClient { default void checkMigrationSettings() { throw new UnsupportedOperationException("未实现迁移能力检查"); } default List migrationWarnings(Repository repository, String sha, long maxFileBytes) { - return List.of("在线下载仍受单文件限制,LFS 实体在线读取能力未扩展;Git 快照 Fork 保留子模块引用,不递归复制依赖"); + return List.of("JSON 文件预览与流式下载分别受大小限制,LFS 实体在线读取能力未扩展;Git 快照 Fork 保留子模块引用,不递归复制依赖"); } Repository createRepository(String name, boolean initialize); diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java index 34815ea..4a5c368 100644 --- a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java @@ -202,6 +202,7 @@ public class RestGiteaClient implements GiteaClient { private long responseLimit(String path, int status) { if (status < 200 || status >= 300) return 64 * 1024; + if (path.matches("/repos/[^/]+/[^/]+/contents-ext/.*")) return 64 * 1024; if (path.contains("/raw/")) return properties.getMaxFileBytes(); if (path.contains("/contents")) return (properties.getMaxFileBytes() + 2) / 3 * 4 + 2 * 1024 * 1024; @@ -261,6 +262,90 @@ public class RestGiteaClient implements GiteaClient { } } + @Override + public String fileDownloadHead(Repository repo, Duration timeout) { + try { + JsonNode branch = parse(request("GET", repoPath(repo) + "/branches/main", null, timeout).body(), false); + String sha = branch.path("commit").path("id").asText(); + if (!sha.matches("[a-f0-9]{40}|[a-f0-9]{64}")) throw FileDownloadInputStream.failed(); + return sha; + } catch (GiteaFailure error) { + if (error.status() == 504) throw FileDownloadInputStream.timedOut(); + throw error; + } + } + + private Duration fileDownloadRemaining(long deadline) { + long nanos = deadline - System.nanoTime(); + if (nanos <= 0) throw FileDownloadInputStream.timedOut(); + return Duration.ofNanos(nanos); + } + + @Override + public FileDownload openFile(Repository repo, String sha, String path, Duration timeout, long maxBytes) { + String normalized = FilePaths.file(path); + String commitSha = FilePaths.sha(sha); + long deadline = System.nanoTime() + timeout.toNanos(); + JsonNode metadata; + try { + metadata = parse(request("GET", repoPath(repo) + "/contents-ext/" + filePath(normalized) + + "?ref=" + commitSha, null, fileDownloadRemaining(deadline)).body(), false); + } catch (GiteaFailure error) { + if (error.status() == 504) throw FileDownloadInputStream.timedOut(); + throw error; + } + JsonNode node = metadata.path("file_contents"); + if (!node.isObject()) { + if (metadata.path("dir_contents").isArray()) + throw BusinessException.badRequest("仅支持普通文件,不支持目录、符号链接或子模块"); + throw FileDownloadInputStream.failed(); + } + if (!node.path("type").isTextual()) throw FileDownloadInputStream.failed(); + if (!"file".equals(node.path("type").asText())) + throw BusinessException.badRequest("仅支持普通文件,不支持目录、符号链接或子模块"); + JsonNode size = node.path("size"); + if (!size.isIntegralNumber() || !size.canConvertToLong() || size.longValue() < 0) + throw FileDownloadInputStream.failed(); + long expected = size.longValue(); + if (expected > maxBytes) throw FileDownloadInputStream.tooLarge(maxBytes); + String base = properties.getGitea().getBaseUrl().replaceAll("/+$", ""); + HttpRequest request = HttpRequest.newBuilder(URI.create(base + "/api/v1" + repoPath(repo) + + "/raw/" + filePath(normalized) + "?ref=" + commitSha)) + .timeout(fileDownloadRemaining(deadline)) + .header("Accept", "application/octet-stream") + .header("Authorization", "token " + properties.getGitea().getToken()) + .GET().build(); + try { + HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofInputStream()); + InputStream input = response.body(); + try { + if (response.statusCode() == 404) + throw new GiteaFailure(404, "NOT_FOUND", "仓库文件或版本不存在", false); + if (response.statusCode() != 200) throw FileDownloadInputStream.failed(); + if (response.headers().firstValue("Content-Length").isPresent()) { + long length = response.headers().firstValueAsLong("Content-Length").orElseThrow(); + if (length > maxBytes) throw FileDownloadInputStream.tooLarge(maxBytes); + if (length != expected) throw FileDownloadInputStream.failed(); + } + fileDownloadRemaining(deadline); + return new FileDownload(new FileDownloadInputStream(input, expected, maxBytes, deadline), expected); + } catch (RuntimeException error) { + try { input.close(); } catch (IOException ignored) { } + if (error instanceof NumberFormatException) throw FileDownloadInputStream.failed(); + throw error; + } + } catch (HttpTimeoutException error) { + throw FileDownloadInputStream.timedOut(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw FileDownloadInputStream.timedOut(); + } catch (IOException error) { + GiteaFailure failure = FileDownloadInputStream.failed(); + failure.initCause(error); + throw failure; + } + } + @Override public String branch(Repository repo, String branchName) { JsonNode branch = @@ -416,7 +501,7 @@ public class RestGiteaClient implements GiteaClient { break; } } - if (large > 0) warnings.add("发布快照有 " + large + " 个文件超过社区单文件读取限制 " + maxFileBytes + " 字节"); + if (large > 0) warnings.add("发布快照有 " + large + " 个文件超过 JSON 文件预览限制 " + maxFileBytes + " 字节,可在流式下载上限内下载"); if (links > 0) warnings.add("发布快照有 " + links + " 个符号链接,在线普通文件读取不支持,Git 快照 Fork 保留链接"); if (modules > 0) warnings.add("发布快照有 " + modules + " 个子模块引用,Git 快照 Fork 保留引用,不递归复制依赖仓库"); return List.copyOf(warnings); diff --git a/vibemake-service/src/test/java/com/linkerbot/vibemake/service/download/FileDownloadServiceTest.java b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/download/FileDownloadServiceTest.java new file mode 100644 index 0000000..abd6e90 --- /dev/null +++ b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/download/FileDownloadServiceTest.java @@ -0,0 +1,79 @@ +package com.linkerbot.vibemake.service.download; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import com.linkerbot.vibemake.bean.entity.Project; +import com.linkerbot.vibemake.service.ProjectService; +import com.linkerbot.vibemake.service.config.FileDownloadProperties; +import com.linkerbot.vibemake.service.gitea.GiteaClient; +import jakarta.validation.Validation; +import jakarta.validation.ValidatorFactory; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicBoolean; + +class FileDownloadServiceTest { + private static final String SHA = "a".repeat(40); + + @Test + void clientCancellationClosesUpstreamAndHeadConsumesSharedTimeBudget() throws Exception { + ProjectService projects = mock(ProjectService.class); + GiteaClient gitea = mock(GiteaClient.class); + FileDownloadProperties properties = new FileDownloadProperties(); + FileDownloadService service = new FileDownloadService(projects, gitea, properties); + Project project = new Project(); + project.setProjectCode("test-project"); + GiteaClient.Repository repository = new GiteaClient.Repository(1, "vibemake", "repo"); + when(projects.readyOwner(42, "test-project")).thenReturn(project); + when(projects.repository(project)).thenReturn(repository); + when(gitea.fileDownloadHead(eq(repository), any())).thenAnswer(invocation -> { + // 模拟获取分支已消耗的预算,后续调用不得重新获得完整的五分钟。 + java.util.concurrent.locks.LockSupport.parkNanos(Duration.ofMillis(5).toNanos()); + return SHA; + }); + AtomicBoolean closed = new AtomicBoolean(); + ByteArrayInputStream input = new ByteArrayInputStream(new byte[] {1, 2, 3}) { + @Override public void close() { closed.set(true); } + }; + when(gitea.openFile(eq(repository), eq(SHA), eq("data.bin"), any(), anyLong())) + .thenReturn(new GiteaClient.FileDownload(input, 3)); + IOException cancelled = new IOException("fixture client disconnected"); + OutputStream output = new OutputStream() { + @Override public void write(int value) throws IOException { throw cancelled; } + @Override public void write(byte[] bytes, int offset, int length) throws IOException { throw cancelled; } + }; + assertThatThrownBy(() -> service.owner(42, "test-project", "data.bin", null, + (name, length) -> output)).isSameAs(cancelled); + assertThat(closed.get()).isTrue(); + ArgumentCaptor headBudget = ArgumentCaptor.forClass(Duration.class); + ArgumentCaptor bodyBudget = ArgumentCaptor.forClass(Duration.class); + verify(gitea).fileDownloadHead(eq(repository), headBudget.capture()); + verify(gitea).openFile(eq(repository), eq(SHA), eq("data.bin"), bodyBudget.capture(), eq(properties.getMaxBytes())); + assertThat(bodyBudget.getValue()).isLessThan(headBudget.getValue()); + verify(gitea, never()).file(any(), anyString(), anyString()); + } + + @Test + void fileDownloadConfigurationRejectsInvalidLimitsAndTimeouts() { + try (ValidatorFactory factory = Validation.buildDefaultValidatorFactory()) { + FileDownloadProperties properties = new FileDownloadProperties(); + assertThat(factory.getValidator().validate(properties)).isEmpty(); + properties.setMaxBytes(0); + assertThat(factory.getValidator().validate(properties)).isNotEmpty(); + properties.setMaxBytes(1); + for (Duration timeout : java.util.List.of(Duration.ZERO, Duration.ofMillis(99), Duration.ofHours(2))) { + properties.setTimeout(timeout); + assertThat(factory.getValidator().validate(properties)).isNotEmpty(); + } + properties.setTimeout(null); + assertThat(factory.getValidator().validate(properties)).isNotEmpty(); + } + } +} diff --git a/vibemake-service/src/test/java/com/linkerbot/vibemake/service/gitea/GiteaFileDownloadTest.java b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/gitea/GiteaFileDownloadTest.java new file mode 100644 index 0000000..b80cc48 --- /dev/null +++ b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/gitea/GiteaFileDownloadTest.java @@ -0,0 +1,271 @@ +package com.linkerbot.vibemake.service.gitea; + +import static org.assertj.core.api.Assertions.*; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkerbot.vibemake.common.BusinessException; +import com.linkerbot.vibemake.service.config.VibemakeProperties; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Timeout(15) +class GiteaFileDownloadTest { + private static final String SHA = "a".repeat(40); + private static final String BASE = "/api/v1/repos/vibemake/repo/"; + private static final String FILE = "目录/模型 空格.gif"; + private static final GiteaClient.Repository REPO = new GiteaClient.Repository(1, "vibemake", "repo"); + private HttpServer server; + private ExecutorService workers; + private RestGiteaClient client; + + @BeforeEach + void start() throws Exception { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + workers = Executors.newVirtualThreadPerTaskExecutor(); + server.setExecutor(workers); + server.start(); + VibemakeProperties config = new VibemakeProperties(); + config.getGitea().setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort()); + config.getGitea().setToken("fixture-token"); + config.setMaxFileBytes(1); + client = new RestGiteaClient(config, new ObjectMapper()); + } + + @AfterEach + void stop() { + server.stop(0); + workers.shutdownNow(); + } + + private void metadata(String json) { + server.createContext(BASE + "contents-ext/" + FILE, exchange -> { + assertThat(exchange.getRequestURI().getRawQuery()).isEqualTo("ref=" + SHA); + assertThat(exchange.getRequestHeaders().getFirst("Authorization")).isEqualTo("token fixture-token"); + byte[] body = json.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream output = exchange.getResponseBody()) { output.write(body); } + }); + } + + private void metadata(long size) { + metadata("{\"file_contents\":{\"type\":\"file\",\"size\":" + size + "}}"); + } + + @ParameterizedTest + @CsvSource({"0,false", "0,true", "3,false", "3,true", "100,false", "100,true"}) + void downloadsAtLimitIncludingEmptyFilesAndChunkedBodies(int size, boolean chunked) throws Exception { + metadata(size); + byte[] expected = new byte[size]; + for (int index = 0; index < size; index++) expected[index] = (byte) index; + server.createContext(BASE + "raw/" + FILE, exchange -> { + assertThat(exchange.getRequestURI().getRawQuery()).isEqualTo("ref=" + SHA); + assertThat(exchange.getRequestHeaders().getFirst("Authorization")).isEqualTo("token fixture-token"); + assertThat(exchange.getRequestHeaders().getFirst("Accept")).isEqualTo("application/octet-stream"); + if (size == 0 && !chunked) exchange.getResponseHeaders().set("Content-Length", "0"); + exchange.sendResponseHeaders(200, chunked ? 0 : size == 0 ? -1 : size); + try (OutputStream output = exchange.getResponseBody()) { output.write(expected); } + }); + try (GiteaClient.FileDownload file = client.openFile(REPO, SHA, FILE, Duration.ofSeconds(3), Math.max(1, size))) { + assertThat(file.contentLength()).isEqualTo(size); + assertThat(file.input().readAllBytes()).isEqualTo(expected); + } + } + + @Test + void streamsMoreThanFiftyMiBBeforeUpstreamFinishesAndPreviewStillHasItsOwnLimit() throws Exception { + int total = 51 * 1024 * 1024; + metadata(total); + CountDownLatch readerStarted = new CountDownLatch(1); + server.createContext(BASE + "raw/" + FILE, exchange -> { + exchange.sendResponseHeaders(200, 0); + try (OutputStream output = exchange.getResponseBody()) { + output.write(new byte[1024]); + output.flush(); + if (!readerStarted.await(3, TimeUnit.SECONDS)) throw new AssertionError("不能预读整个文件"); + byte[] block = new byte[64 * 1024]; + for (int written = 1024; written < total; written += block.length) + output.write(block, 0, Math.min(block.length, total - written)); + } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + }); + try (GiteaClient.FileDownload file = client.openFile(REPO, SHA, FILE, Duration.ofSeconds(8), total)) { + assertThat(file.input().read()).isZero(); + readerStarted.countDown(); + assertThat(file.input().transferTo(OutputStream.nullOutputStream())).isEqualTo(total - 1L); + } finally { readerStarted.countDown(); } + server.createContext(BASE + "contents/" + FILE, exchange -> { + byte[] body = ("{\"type\":\"file\",\"size\":" + total + "}").getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream output = exchange.getResponseBody()) { output.write(body); } + }); + assertThatThrownBy(() -> client.file(REPO, SHA, FILE)).isInstanceOfSatisfying(BusinessException.class, + error -> assertThat(error.code()).isEqualTo("FILE_TOO_LARGE")); + } + + @Test + void oversizedMetadataStopsBeforeRawRequestAndReportsDownloadLimit() { + metadata(101); + AtomicInteger calls = new AtomicInteger(); + server.createContext(BASE + "raw/" + FILE, exchange -> { calls.incrementAndGet(); exchange.close(); }); + assertThatThrownBy(() -> client.openFile(REPO, SHA, FILE, Duration.ofSeconds(2), 100)) + .isInstanceOfSatisfying(GiteaFailure.class, error -> { + assertThat(error.code()).isEqualTo("FILE_TOO_LARGE"); + assertThat(error.getMessage()).contains("100 字节"); + }); + assertThat(calls.get()).isZero(); + } + + @Test + void metadataResponseHasItsOwnSmallLimit() { + metadata("{\"unexpected_content\":\"" + "x".repeat(100000) + "\"}"); + assertThatThrownBy(() -> client.openFile(REPO, SHA, FILE, Duration.ofSeconds(2), 1024L * 1024 * 1024)) + .isInstanceOfSatisfying(GiteaFailure.class, + error -> assertThat(error.code()).isEqualTo("GITEA_RESPONSE_TOO_LARGE")); + } + + @Test + void previewDirectoryNamedContentsExtDoesNotUseMetadataLimit() throws Exception { + String path = "contents-ext/data.bin"; + server.createContext(BASE + "contents/" + path, exchange -> { + byte[] body = ("{\"type\":\"file\",\"size\":1,\"encoding\":\"base64\",\"content\":\"AQ==\",\"padding\":\"" + + "x".repeat(70000) + "\"}").getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream output = exchange.getResponseBody()) { output.write(body); } + }); + assertThat(client.file(REPO, SHA, path).bytes()).containsExactly((byte) 1); + } + + @Test + void metadataAndRawBodyShareOneDeadline() { + CountDownLatch release = new CountDownLatch(1); + server.createContext(BASE + "contents-ext/" + FILE, exchange -> { + java.util.concurrent.locks.LockSupport.parkNanos(Duration.ofMillis(650).toNanos()); + byte[] body = "{\"file_contents\":{\"type\":\"file\",\"size\":1}}".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream output = exchange.getResponseBody()) { output.write(body); } + }); + server.createContext(BASE + "raw/" + FILE, exchange -> { + exchange.sendResponseHeaders(200, 0); + try (OutputStream output = exchange.getResponseBody()) { + output.flush(); + release.await(650, TimeUnit.MILLISECONDS); + output.write(new byte[] {1}); + } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + catch (java.io.IOException expected) { /* 共享预算到期后客户端关闭。 */ } + }); + try { + assertThatThrownBy(() -> { + try (GiteaClient.FileDownload file = client.openFile(REPO, SHA, FILE, Duration.ofSeconds(1), 100)) { + file.input().transferTo(OutputStream.nullOutputStream()); + } + }).isInstanceOfSatisfying(GiteaFailure.class, + error -> assertThat(error.code()).isEqualTo("FILE_DOWNLOAD_TIMEOUT")); + } finally { release.countDown(); } + } + + @ParameterizedTest + @CsvSource({"100,101,false,FILE_TOO_LARGE", "3,4,false,GITEA_ERROR", + "3,2,false,GITEA_ERROR", "3,4,true,GITEA_ERROR", "3,2,true,GITEA_ERROR", + "100,101,true,FILE_TOO_LARGE"}) + void checksHeadersAndActualByteCount(long expected, int actual, boolean chunked, String code) { + metadata(expected); + server.createContext(BASE + "raw/" + FILE, exchange -> { + exchange.sendResponseHeaders(200, chunked ? 0 : actual); + try (OutputStream output = exchange.getResponseBody()) { output.write(new byte[actual]); } + }); + assertThatThrownBy(() -> { + try (GiteaClient.FileDownload file = client.openFile(REPO, SHA, FILE, Duration.ofSeconds(2), 100)) { + file.input().transferTo(OutputStream.nullOutputStream()); + } + }).isInstanceOfSatisfying(GiteaFailure.class, error -> assertThat(error.code()).isEqualTo(code)); + } + + @ParameterizedTest + @ValueSource(strings = {"dir", "symlink", "submodule"}) + void rejectsNonRegularFiles(String type) { + metadata(type.equals("dir") ? "{\"dir_contents\":[]}" + : "{\"file_contents\":{\"type\":\"" + type + "\",\"size\":0}}"); + assertThatThrownBy(() -> client.openFile(REPO, SHA, FILE, Duration.ofSeconds(2), 100)) + .isInstanceOfSatisfying(BusinessException.class, error -> assertThat(error.status()).isEqualTo(400)); + } + + @ParameterizedTest + @ValueSource(strings = {"{}", "null", "[]", "{\"file_contents\":{\"type\":\"file\"}}", + "{\"file_contents\":{\"type\":\"file\",\"size\":-1}}", + "{\"file_contents\":{\"type\":\"file\",\"size\":1.5}}", + "{\"file_contents\":{\"type\":\"file\",\"size\":\"3\"}}"}) + void malformedMetadataIsAnUpstreamError(String json) { + metadata(json); + assertThatThrownBy(() -> client.openFile(REPO, SHA, FILE, Duration.ofSeconds(2), 100)) + .isInstanceOfSatisfying(GiteaFailure.class, error -> assertThat(error.status()).isEqualTo(502)); + } + + @ParameterizedTest + @CsvSource({"true,302", "false,302", "true,404", "false,404", "true,500", "false,500", + "false,401", "false,403"}) + void errorsAndRedirectsDoNotBecomeFileContents(boolean metadataError, int status) { + AtomicInteger redirected = new AtomicInteger(); + server.createContext("/redirected", exchange -> { redirected.incrementAndGet(); exchange.close(); }); + if (!metadataError) metadata(3); + server.createContext(BASE + (metadataError ? "contents-ext/" : "raw/") + FILE, exchange -> { + exchange.getResponseHeaders().set("Location", "/redirected"); + exchange.sendResponseHeaders(status, -1); + exchange.close(); + }); + assertThatThrownBy(() -> client.openFile(REPO, SHA, FILE, Duration.ofSeconds(2), 100)) + .isInstanceOfSatisfying(GiteaFailure.class, error -> { + assertThat(error.status()).isEqualTo(status == 404 ? 404 : 502); + assertThat(error.uncertain()).isFalse(); + }); + assertThat(redirected.get()).isZero(); + } + + @ParameterizedTest + @ValueSource(strings = {"main", "metadata", "headers", "body"}) + void timeoutCoversEveryUpstreamStage(String stage) { + CountDownLatch release = new CountDownLatch(1); + if (stage.equals("headers") || stage.equals("body")) metadata(100); + String path = switch (stage) { + case "main" -> "branches/main"; + case "metadata" -> "contents-ext/" + FILE; + default -> "raw/" + FILE; + }; + server.createContext(BASE + path, exchange -> { + try { + if (stage.equals("body")) { + exchange.sendResponseHeaders(200, 0); + exchange.getResponseBody().write(new byte[] {1}); + exchange.getResponseBody().flush(); + } + release.await(5, TimeUnit.SECONDS); + } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + finally { exchange.close(); } + }); + long started = System.nanoTime(); + try { + assertThatThrownBy(() -> { + if (stage.equals("main")) client.fileDownloadHead(REPO, Duration.ofMillis(350)); + else try (GiteaClient.FileDownload file = client.openFile(REPO, SHA, FILE, Duration.ofMillis(350), 100)) { + file.input().transferTo(OutputStream.nullOutputStream()); + } + }).isInstanceOfSatisfying(GiteaFailure.class, + error -> assertThat(error.code()).isEqualTo("FILE_DOWNLOAD_TIMEOUT")); + assertThat(Duration.ofNanos(System.nanoTime() - started)).isLessThan(Duration.ofSeconds(3)); + } finally { release.countDown(); } + } +}