diff --git a/.env.example b/.env.example index 8d92cfe..3f79f0f 100644 --- a/.env.example +++ b/.env.example @@ -90,3 +90,8 @@ RATE_COVER_SIGN_PER_MINUTE=30 RATE_URDF_SIGN_PER_MINUTE=20 RATE_COMMENT_PER_MINUTE=10 RATE_LIKE_PER_MINUTE=60 + +# README 普通外链图片自动转存;动态徽章保留原链。 +README_IMAGES_ENABLED=true +README_IMAGE_MAX_BYTES=20971520 +README_IMAGE_CONCURRENCY=2 diff --git a/README.md b/README.md index 1d34260..9cf621f 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ prod 启动时检查数据库连接、用户名、密码、Gitea 地址和 Token | API 跨域来源 | `https://dev-vibemake.robotquan.com` | `https://vibemake.robotquan.com` | | OSS 封面目录 | vibemake_dev/project-covers/ | vibemake/project-covers/ | | OSS URDF 目录 | vibemake_dev/urdf/{projectCode}/{batchCode}/ | vibemake/urdf/{projectCode}/{batchCode}/ | +| OSS README 图片目录 | vibemake_dev/readme-images/ | vibemake/readme-images/ | | 连接池 | 最少 1、最多 10 个连接 | 最少 1、最多 20 个连接 | | 项目日志级别 | DEBUG | INFO | | 日志目录 | logs,可用 LOG_PATH 覆盖 | logs,可用 LOG_PATH 覆盖 | @@ -220,6 +221,31 @@ Fork 不受普通在线文件读取的 50 MiB 限制,文件浏览、下载和 ## 项目短编码与 GitHub 初始化导入 +### README 外链图片转存 OSS + +GitHub 后台迁移和 Excel 命令行导入在分支适配后自动检查根目录 `README.md`(文件名不区分大小写)。支持 Markdown 图片、引用式图片、HTML `img src`、`srcset` 和 `picture` 图片;仓库相对路径、代码示例、普通超链接及已配置的 OSS/CDN 地址保持原样。动态徽章(shields.io、GitHub Actions、ROS 构建状态、常见覆盖率徽章)保留原链,普通静态 SVG 通过安全校验后可以转存;仅外部 DTD 声明会移除,脚本、内部实体和远程资源引用仍禁止。其他语言的 README 或子目录文档不在本次处理范围。 + +设置 `README_IMAGES_ENABLED=false` 可以暂停转存;默认启用,单图上限 `README_IMAGE_MAX_BYTES=20971520`,图片并发 `README_IMAGE_CONCURRENCY=2`(1~4)。可在配置的 `vibemake.readme-images.preserve-url-patterns` 中追加需要保留的完整 URL 正则。图片只下载公开 HTTP/HTTPS 地址,不携带业务凭据;单图总预算 45 秒,每次 DNS 解析及重定向都禁止内网地址。JPG、PNG、GIF、WebP 和安全 SVG 按内容摘要写入 `{环境根目录}/readme-images/{SHA256}.{扩展名}`,不覆盖已有对象;上传核验并从公开地址读取核验成功后才替换 README。RAM 账号需要对应前缀的 PutObject/GetObject 权限,资源需要公开可读或通过公开 CDN 访问。 + +存量项目在项目根目录运行: + +```bash +# 仅扫描数据库与仓库,不下载外链、不上传图片、不修改项目。 +bash scripts/migrate-readme-images.sh --dry-run +# 先处理一个含待转存图片的项目。 +bash scripts/migrate-readme-images.sh --execute --limit 1 +# 同一状态目录断点续跑其余项目。 +bash scripts/migrate-readme-images.sh --execute +# 指定项目重新尝试仍失败的图片,使用新的状态目录保存独立收据。 +bash scripts/migrate-readme-images.sh --execute --project-code 项目编码 --state-dir logs/readme-image-retry +``` + +默认状态目录 `logs/readme-image-migration` 绑定 Gitea、组织、OSS Bucket、环境目录和公开基础地址,禁止跨环境复用;同时只允许一个进程使用。保存精确预览、原 README 备份、原发布 SHA、每一步操作收据和结果报告,目录/文件权限分别为 0700/0600,写入原子替换并持久化。脚本使用 `java-with-env.py` 安全读取 `.env` 字面量,不进行 shell 求值;含 `&` 的 JDBC 地址可正常读取,文件中的配置覆盖同名进程变量,与原有启动脚本一致。不会修改 `.env` 或输出配置值。 + +未删除且仓库 READY 的项目才进入批处理。main 与原发布版本一致时复用一个图片提交;不一致时从原发布 SHA 创建 `maintenance/readme-oss/...` 维护分支,确认仅 README 变化后更新发布引用。维护分支保留以确保快照可下载和 Fork;不将 main 上其他未发布代码带入社区,不改变撤回/下架状态或发布时间。GitHub 后台迁移仍由管理员手动发布,Excel 导入仍自动发布处理后的版本,来源 SHA 与图片提交 SHA 分开保存,兼容历史迁移收据。 + +图片下载失败、格式不支持或 OSS 不可用时保留原链并记录告警,其余图片继续;Git 写入结果不明确则保持待核对状态,续跑只查收据,不重发提交。报告的 `failed` 是保留原链的图片引用数。恢复业务展示可在项目锁中对照备份条件回写原发布 SHA;恢复 main 需基于当前版本提交 README 的反向修改,不能强制重置分支。不要删除共享 OSS 对象或维护分支。一般日志只记录项目编码、来源主机、阶段和错误码,含完整图片地址的原文只保存在受限备份中。 + ### 历史外链封面转存 OSS `scripts/migrate-project-covers.sh` 读取本项目 `.env`,将数据库现有外链图片下载并转存至当前环境 OSS,上传并核验成功后才更新 `cover_url`。配置 `DB_URL`、`DB_USERNAME`、`DB_PASSWORD`、`OSS_REGION`、`OSS_BUCKET`、`OSS_ACCESS_KEY_ID`、`OSS_ACCESS_KEY_SECRET`;带 `&` 的 JDBC 地址必须加引号。环境根目录沿用 test 的 `vibemake_dev` 或 prod 的 `vibemake`。 @@ -251,7 +277,7 @@ bash scripts/migrate-project-covers.sh --execute 外部封面保留原 URL 和 GIF,不转存 OSS;当前环境的平台封面统一转换为相对路径。描述由原中文简介和技术资料组合,不调用 AI。轮式、四足、人形分别映射对应类型,机械臂映射 ROBOTIC_ARM,其他和硬件映射 GENERAL,原始分类保留在描述里。 -脚本加载本项目 `.env`,依赖 Java 21、Maven,以及正式执行时可用的 MySQL 和 Gitea。独立导入模块不启动 Web 服务,也不加载登录、AI 或 OSS 客户端;只读取资源地址配置,不需要 OSS 凭据。test/prod 根目录与在线应用一致,历史公开地址使用 `OSS_LEGACY_PUBLIC_BASE_URLS`。Gitea 技术账号需具备组织所有者及仓库写权限,实例需开启迁移、HTTP Git 和 LFS。`GITHUB_TOKEN` 可选,仅用于 GitHub 元数据 API;API 限流时读取同一公开仓库的 GitHub 网页,交叉核验仓库 ID、规范地址、公开状态和默认分支,缺失或不一致则预检失败。网页和图片请求均不携带凭据。`GITEA_TOKEN` 仅访问配置的 Gitea,不能代替 `GITHUB_TOKEN`;切勿把 Token 放进 URL 或命令参数。 +脚本加载本项目 `.env`,依赖 Java 21、Maven,以及正式执行时可用的 MySQL 和 Gitea。独立导入模块不启动 Web 服务,也不加载登录或 AI。资源地址转换无需 OSS 凭据;README 图片转存按需初始化 OSS 客户端,未配置凭据时保留原链并产生告警。test/prod 根目录与在线应用一致,历史公开地址使用 `OSS_LEGACY_PUBLIC_BASE_URLS`。Gitea 技术账号需具备组织所有者及仓库写权限,实例需开启迁移、HTTP Git 和 LFS。`GITHUB_TOKEN` 可选,仅用于 GitHub 元数据 API;API 限流时读取同一公开仓库的 GitHub 网页,交叉核验仓库 ID、规范地址、公开状态和默认分支,缺失或不一致则预检失败。网页和图片请求均不携带凭据。`GITEA_TOKEN` 仅访问配置的 Gitea,不能代替 `GITHUB_TOKEN`;切勿把 Token 放进 URL 或命令参数。 ```bash IMPORT_EXCEL='/绝对路径/项目.xlsx' diff --git a/docs/admin-api.md b/docs/admin-api.md index 4bb7d5e..83db4bf 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -1076,7 +1076,7 @@ URDF 缺失或大小不符时拒绝创建,不留下项目和任务。收到 20 ```ts type ImportStatus = 'QUEUED' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'REVIEW'; type ImportPhase = 'QUEUED' | 'CLEANING' | 'PRECHECK' | 'MIGRATING' | 'VERIFYING' | 'MIGRATED' - | 'NORMALIZING' | 'BRANCH_READY' | 'LINKED'; + | 'NORMALIZING' | 'BRANCH_READY' | 'README_IMAGES' | 'LINKED'; interface TaskView { taskCode: string; @@ -1144,8 +1144,11 @@ interface TaskView { | MIGRATED | 仓库迁移已确认 | | NORMALIZING | 适配社区主分支 | | BRANCH_READY | 主分支已准备 | +| README_IMAGES | 转存 README 外链图片并核对提交 | | LINKED | 已关联社区项目 | +README 图片转存自动在主分支适配后执行,支持 Markdown、HTML 图片及备用图片地址;仓库内图片和动态徽章保持原样。单图失败保留原链,原因追加到 `warnings`,迁移仍可完成。README 提交结果不明时任务进入 `REVIEW`,重试只核对原操作收据,不重新迁移或重发提交。后台迁移完成后仍需管理员手动发布。 + 不展示百分比进度,接口没有 progress、duration 字段。canRetry 由后端判断,前端不要仅按 status/phase 猜测是否可重试;重试仍会校验项目最新状态,可能返回 409。retryMessage 是重试确认框或无法重试时的说明,正常执行中/成功时为 null。耗时可根据 startedAt、finishedAt 计算,空值显示 `—`。任务查询 HTTP 200、code=OK 只表示查询成功,必须再判断 data.status。 ### 9.5 轮询示例 diff --git a/docs/admin-database.md b/docs/admin-database.md index 858520a..b6f5611 100644 --- a/docs/admin-database.md +++ b/docs/admin-database.md @@ -277,3 +277,5 @@ ALTER TABLE repository_write_operation ADD COLUMN actor_user_code VARCHAR(32) NULL COMMENT '实际发起推送的网站用户业务编码', ADD COLUMN git_token_code CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NULL COMMENT '本次推送使用的Git凭据业务编码,不保存明文'; ``` + +README 外链图片转存沿用 `phase` 的 VARCHAR 字段,增加 `README_IMAGES` 阶段,无需修改表结构。`receipt_json` 在原仓库收据上追加可选 `readmeImages` 进度,包含处理基准、分支、操作编码、内容摘要和结果 SHA;兼容缺少该字段的历史收据,不保存图片、README 正文或认证凭据。`target_sha` 始终保留 GitHub 原始迁移 SHA,图片提交 SHA 单独记录,续跑不能按原 SHA 再次归一化已处理的 main。 diff --git a/scripts/import-github-projects.sh b/scripts/import-github-projects.sh index e872cb9..02d7ada 100644 --- a/scripts/import-github-projects.sh +++ b/scripts/import-github-projects.sh @@ -1,11 +1,6 @@ #!/usr/bin/env bash set -euo pipefail VIBEMAKE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [[ -f "$VIBEMAKE_ROOT/.env" ]]; then - set -a - source "$VIBEMAKE_ROOT/.env" - set +a -fi cd "$VIBEMAKE_ROOT" mvn -q -pl vibemake-import -am -DskipTests package -exec java -Djava.awt.headless=true -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8 -jar vibemake-import/target/vibemake-import-0.0.1-SNAPSHOT.jar "$@" +exec python3 scripts/java-with-env.py -Djava.awt.headless=true -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8 -jar vibemake-import/target/vibemake-import-0.0.1-SNAPSHOT.jar "$@" diff --git a/scripts/java-with-env.py b/scripts/java-with-env.py new file mode 100755 index 0000000..1311530 --- /dev/null +++ b/scripts/java-with-env.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""读取 dotenv 字面量并启动 Java,不将配置交给 shell 求值。""" +import os +import pathlib +import re +import sys + + +def load_environment(path, inherited): + values = dict(inherited) + if path.exists(): + for number, line in enumerate(path.read_text(encoding="utf-8-sig").splitlines(), 1): + if not line.strip() or line.lstrip().startswith("#"): + continue + match = re.fullmatch(r"\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=(.*)", line) + if not match: + raise ValueError(f"环境文件第 {number} 行格式不合法") + value = match[2].strip() + if value[:1] in ("'", '"'): + if len(value) < 2 or value[-1] != value[0]: + raise ValueError(f"环境文件第 {number} 行引号未闭合") + value = value[1:-1] + else: + value = re.split(r"\s+#", value, maxsplit=1)[0].rstrip() + values[match[1]] = value + return values + + +if __name__ == "__main__": + try: + root = pathlib.Path(__file__).resolve().parent.parent + environment = load_environment(pathlib.Path(os.environ.get("VIBEMAKE_ENV_FILE", root / ".env")), os.environ) + if len(sys.argv) < 2: + raise ValueError("缺少 Java 启动参数") + os.execvpe("java", ["java", *sys.argv[1:]], environment) + except (ValueError, OSError) as error: + # 不回显配置内容或底层异常文本。 + print(f"Java 启动失败 type={type(error).__name__}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/migrate-project-covers.sh b/scripts/migrate-project-covers.sh index ac61703..b7b932a 100644 --- a/scripts/migrate-project-covers.sh +++ b/scripts/migrate-project-covers.sh @@ -1,14 +1,9 @@ #!/usr/bin/env bash set -euo pipefail VIBEMAKE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [[ -f "$VIBEMAKE_ROOT/.env" ]]; then - set -a - source "$VIBEMAKE_ROOT/.env" - set +a -fi cd "$VIBEMAKE_ROOT" mvn -q -pl vibemake-import -am -DskipTests package -exec java -Djava.awt.headless=true -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8 \ +exec python3 scripts/java-with-env.py -Djava.awt.headless=true -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8 \ -Dlogging.level.root=WARN -Dlogging.level.org.springframework.jdbc=INFO \ -Dlogging.level.org.apache.http=WARN -Ddebug=false \ -jar vibemake-import/target/vibemake-import-0.0.1-SNAPSHOT.jar covers "$@" diff --git a/scripts/migrate-readme-images.sh b/scripts/migrate-readme-images.sh new file mode 100755 index 0000000..2d354ae --- /dev/null +++ b/scripts/migrate-readme-images.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +VIBEMAKE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$VIBEMAKE_ROOT" +mvn -q -pl vibemake-import -am -DskipTests package +exec python3 scripts/java-with-env.py -Djava.awt.headless=true -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8 \ + -Dlogging.level.root=WARN -Dlogging.level.org.springframework.jdbc=INFO \ + -Dlogging.level.org.apache.http=WARN -Dlogging.level.com.aliyun.oss=OFF -Ddebug=false \ + -jar vibemake-import/target/vibemake-import-0.0.1-SNAPSHOT.jar readme-images "$@" diff --git a/scripts/tests/test_java_with_env.py b/scripts/tests/test_java_with_env.py new file mode 100644 index 0000000..8b27a72 --- /dev/null +++ b/scripts/tests/test_java_with_env.py @@ -0,0 +1,28 @@ +import importlib.util +import pathlib +import tempfile +import unittest + +SPEC = importlib.util.spec_from_file_location("java_with_env", pathlib.Path(__file__).parents[1] / "java-with-env.py") +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class EnvironmentTest(unittest.TestCase): + def test_literals_preserve_jdbc_ampersands_and_never_execute_shell(self): + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / ".env" + path.write_text("DB_URL=jdbc:mysql://example/db?a=1&b=2\nDB_PASSWORD='literal$(command)`text`'\nEMPTY=\n") + values = MODULE.load_environment(path, {"DB_PASSWORD": "inherited", "KEEP": "yes"}) + self.assertEqual(values["DB_URL"], "jdbc:mysql://example/db?a=1&b=2") + self.assertEqual(values["DB_PASSWORD"], "literal$(command)`text`") + self.assertEqual(values["KEEP"], "yes") + self.assertEqual(values["EMPTY"], "") + + def test_invalid_lines_are_rejected_without_echoing_secrets(self): + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / ".env" + path.write_text("PASSWORD='secret\n") + with self.assertRaisesRegex(ValueError, "第 1 行") as caught: + MODULE.load_environment(path, {}) + self.assertNotIn("secret", str(caught.exception)) diff --git a/vibemake-adminapi/src/main/resources/application.yml b/vibemake-adminapi/src/main/resources/application.yml index 22277ac..c85c365 100644 --- a/vibemake-adminapi/src/main/resources/application.yml +++ b/vibemake-adminapi/src/main/resources/application.yml @@ -67,6 +67,10 @@ vibemake: worker-enabled: ${ADMIN_IMPORT_WORKER_ENABLED:true} migration-timeout: ${ADMIN_MIGRATION_TIMEOUT:30m} import-concurrency: ${ADMIN_IMPORT_CONCURRENCY:2} + readme-images: + enabled: ${README_IMAGES_ENABLED:true} + max-bytes: ${README_IMAGE_MAX_BYTES:20971520} + concurrency: ${README_IMAGE_CONCURRENCY:2} oss: enabled: ${OSS_ENABLED:true} region: ${OSS_REGION:cn-beijing} diff --git a/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/AdminWorkflowTest.java b/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/AdminWorkflowTest.java index 70280b7..fa4c3f2 100644 --- a/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/AdminWorkflowTest.java +++ b/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/AdminWorkflowTest.java @@ -68,6 +68,14 @@ class AdminWorkflowTest { return new FakeOssObjectStore(); } + @Bean + @Primary + com.linkerbot.vibemake.service.readme.ReadmeImages fakeReadmeImages(OssProperties oss) { + return new com.linkerbot.vibemake.service.readme.ReadmeImages( + new com.linkerbot.vibemake.service.config.ReadmeImageProperties(), oss, + url -> "https://assets.example.invalid/readme-images/image.png"); + } + @Bean @Primary GitHubSourceChecks fakeSources() { @@ -1742,6 +1750,29 @@ class AdminWorkflowTest { .path("data").path("status").asText()).isEqualTo("SUCCEEDED"); } + @Test + void githubMigrationReplacesReadmeImagesAndPreservesOriginalShaWithoutPublishing() throws Exception { + JsonNode task = createImport(); + gitea.migrationReadme = "# project\n![photo](https://example.com/a.png)\n![local](./a.png)\n![badge](https://img.shields.io/badge/build-pass-green.svg)\n"; + imports.run(task.path("taskCode").asText()); + Project project = projects.selectOne(new com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper() + .eq(Project::getProjectCode, task.path("projectCode").asText())); + ProjectImportTask finished = tasks.selectOne(new com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper() + .eq(ProjectImportTask::getTaskCode, task.path("taskCode").asText())); + com.linkerbot.vibemake.service.gitea.GiteaClient.Repository repository = new com.linkerbot.vibemake.service.gitea.GiteaClient.Repository(project.getGiteaRepoId(), project.getGiteaOwner(), project.getGiteaRepoName()); + com.linkerbot.vibemake.service.readme.ReadmeRepository.State state = com.linkerbot.vibemake.service.readme.ReadmeMigrationReceipt.state(json, finished.getReceiptJson()); + assertThat(finished.getStatus()).isEqualTo("SUCCEEDED"); + assertThat(state.baseSha).isEqualTo(finished.getTargetSha()).isNotEqualTo(state.resultSha); + assertThat(state.resultSha).isEqualTo(gitea.head(repository)); + assertThat(project.getPublishStatus()).isEqualTo("UNPUBLISHED"); + assertThat(new String(gitea.file(repository, state.resultSha, "README.md").bytes(), java.nio.charset.StandardCharsets.UTF_8)) + .contains("https://assets.example.invalid/readme-images/image.png", "![local](./a.png)", "https://img.shields.io/badge/build-pass-green.svg"); + int committed = gitea.commitCalls; + imports.run(task.path("taskCode").asText()); + assertThat(gitea.migrations).isEqualTo(1); + assertThat(gitea.commitCalls).isEqualTo(committed); + } + @Test void receiptSurvivesAssociationFailureAndResumesWithoutSecondMigration() throws Exception { JsonNode task = createImport(); @@ -1757,7 +1788,7 @@ class AdminWorkflowTest { .eq(ProjectImportTask::getTaskCode, code)); assertThat(failed.getStatus()).isEqualTo("REVIEW"); assertThat(failed.getReceiptJson()).isNotNull(); - assertThat(failed.getPhase()).isEqualTo("BRANCH_READY"); + assertThat(failed.getPhase()).isEqualTo("README_IMAGES"); assertThat( jdbc.queryForObject( "SELECT repo_status FROM project WHERE project_code=?", diff --git a/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/FakeGiteaClient.java b/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/FakeGiteaClient.java index 756182b..45b0763 100644 --- a/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/FakeGiteaClient.java +++ b/vibemake-adminapi/src/test/java/com/linkerbot/vibemake/adminapi/FakeGiteaClient.java @@ -21,6 +21,7 @@ final class FakeGiteaClient implements GiteaClient { private final AtomicLong sequence = new AtomicLong(); String lastAuthorCode; int migrations; + String migrationReadme; boolean migrationTimeout; boolean migrationRejected; boolean timeoutNextDelete; @@ -55,6 +56,7 @@ final class FakeGiteaClient implements GiteaClient { migrationNames.clear(); mainReady.clear(); migrations = 0; + migrationReadme = null; deletions = 0; migrationDescriptions.clear(); migrationTimeout = false; @@ -223,6 +225,8 @@ final class FakeGiteaClient implements GiteaClient { migrations++; migrationNames.add(name); Repository repository = createRepository(name, true); + if (migrationReadme != null) repo(repository).snapshots.get(repo(repository).head) + .put("README.md", migrationReadme.getBytes(java.nio.charset.StandardCharsets.UTF_8)); RemoteRepository remote = new RemoteRepository( repository.id(), "vibemake", name, true, false, false, source, "master"); diff --git a/vibemake-api/src/main/resources/application.yml b/vibemake-api/src/main/resources/application.yml index 7c34dc3..36820e4 100644 --- a/vibemake-api/src/main/resources/application.yml +++ b/vibemake-api/src/main/resources/application.yml @@ -66,6 +66,10 @@ vibemake: cad-read: ${RATE_CAD_READ_PER_MINUTE:60} comment: ${RATE_COMMENT_PER_MINUTE:10} like: ${RATE_LIKE_PER_MINUTE:60} + readme-images: + enabled: ${README_IMAGES_ENABLED:true} + max-bytes: ${README_IMAGE_MAX_BYTES:20971520} + concurrency: ${README_IMAGE_CONCURRENCY:2} oss: enabled: ${OSS_ENABLED:true} region: ${OSS_REGION:cn-beijing} diff --git a/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/DatabaseConfiguration.java b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/DatabaseConfiguration.java index 3490440..8bb8b34 100644 --- a/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/DatabaseConfiguration.java +++ b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/DatabaseConfiguration.java @@ -15,7 +15,7 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -/** 仅加载资源路径配置,不初始化在线安全、AI 或 OSS 客户端。 */ +/** 仅加载导入所需服务,不启动在线安全或 AI;README OSS 客户端按需初始化。 */ @Configuration(proxyBeanMethods = false) @EnableAutoConfiguration @MapperScan("com.linkerbot.vibemake.service.mapper") @@ -28,6 +28,10 @@ import org.springframework.context.annotation.Import; ProjectLocks.class, OssProperties.class, OssResourcePaths.class, + com.linkerbot.vibemake.service.config.ReadmeImageProperties.class, + com.linkerbot.vibemake.service.readme.ImageOssUploader.class, + com.linkerbot.vibemake.service.readme.ReadmeImages.class, + com.linkerbot.vibemake.service.readme.ReadmeRepository.class, ProjectResourceFiles.class }) public class DatabaseConfiguration {} diff --git a/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportApplication.java b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportApplication.java index e81157f..73f9fe5 100644 --- a/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportApplication.java +++ b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportApplication.java @@ -91,6 +91,11 @@ public final class ImportApplication { } public static void main(String[] args) { + if (args.length > 0 && "readme-images".equals(args[0])) { + int status = com.linkerbot.vibemake.importer.readme.ReadmeMigrationApplication.runCli(Arrays.copyOfRange(args, 1, args.length)); + if (status != 0) System.exit(status); + return; + } if (args.length > 0 && "covers".equals(args[0])) { int status = com.linkerbot.vibemake.importer.cover.CoverMigrationApplication.runCli( Arrays.copyOfRange(args, 1, args.length)); @@ -234,7 +239,8 @@ public final class ImportApplication { context.getBean(OssResourcePaths.class), store, owner, - options.migrationTimeout()).locks(context.getBean(com.linkerbot.vibemake.service.support.ProjectLocks.class)); + options.migrationTimeout()).locks(context.getBean(com.linkerbot.vibemake.service.support.ProjectLocks.class)) + .readmes(context.getBean(com.linkerbot.vibemake.service.readme.ReadmeRepository.class)); try (ExecutorService executor = Executors.newFixedThreadPool(options.concurrency())) { try { diff --git a/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportCoordinator.java b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportCoordinator.java index 643419a..fdc1772 100644 --- a/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportCoordinator.java +++ b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportCoordinator.java @@ -34,6 +34,12 @@ public final class ImportCoordinator { private final User owner; private final Duration timeout; private final String batch; + private com.linkerbot.vibemake.service.readme.ReadmeRepository readmes; + + public ImportCoordinator readmes(com.linkerbot.vibemake.service.readme.ReadmeRepository readmes) { + this.readmes = readmes; + return this; + } private com.linkerbot.vibemake.service.support.ProjectLocks sharedLocks = new com.linkerbot.vibemake.service.support.ProjectLocks(); @@ -46,7 +52,7 @@ public final class ImportCoordinator { return entry.phase == Phase.REVIEW && entry.receipt != null && entry.targetSha != null - && "NORMALIZING".equals(entry.failedPhase) + && ("NORMALIZING".equals(entry.failedPhase) || entry.readmeImages != null) && !"SOURCE_CHANGED".equals(entry.errorCode) && !"DUPLICATE_SOURCE".equals(entry.errorCode); } @@ -160,14 +166,14 @@ public final class ImportCoordinator { if (!Objects.equals( project.getGiteaRepoId(), entry.receipt == null ? null : entry.receipt.id()) - || !Objects.equals(project.getPublishedCommitSha(), entry.targetSha)) + || !Objects.equals(project.getPublishedCommitSha(), entry.readmeImages == null ? entry.targetSha : entry.readmeImages.resultSha)) throw new ImportFailure("PUBLICATION_CONFLICT", "已发布项目与导入收据不一致"); store.phase(entry, Phase.PUBLISHED); return; } } - if (entry.phase == Phase.REVIEW) { + if (entry.phase == Phase.REVIEW && entry.readmeImages == null) { RemoteRepository confirmed = gitea.inspect(repoName); validateRemote(entry, confirmed); boolean applied = @@ -205,7 +211,13 @@ public final class ImportCoordinator { gitea, config.getGitea().getOwner()); com.linkerbot.vibemake.service.migration.RepositoryMigration.Result migrated; try { - migrated = + if (entry.readmeImages != null) { + RemoteRepository confirmed = engine.verifyReceipt(entry.source.url(), repoName, entry.receipt); + if (readmes == null) throw new ImportFailure("README_COMPONENT_MISSING", "缺少 README 处理组件"); + readmes.resume(entry.projectCode, confirmed.repository(), entry.readmeImages, state -> checkpointReadme(entry, state)); + sharedLocks.confirmKnownResult(entry.projectCode); + migrated = new com.linkerbot.vibemake.service.migration.RepositoryMigration.Result(confirmed, entry.targetSha); + } else migrated = sharedLocks.write( entry.projectCode, () -> { @@ -256,6 +268,17 @@ public final class ImportCoordinator { RemoteRepository remote = migrated.remote(); String sourceSha = migrated.sha(); + if (readmes != null) { + com.linkerbot.vibemake.service.readme.ReadmeRepository.Prepared prepared = entry.readmeImages == null + ? readmes.prepare(entry.projectCode, remote.repository(), sourceSha, true) : null; + String baseSha = sourceSha; + com.linkerbot.vibemake.service.readme.ReadmeRepository.State state = sharedLocks.write(entry.projectCode, + () -> readmes.apply(entry.projectCode, remote.repository(), baseSha, "main", prepared, entry.readmeImages, + next -> checkpointReadme(entry, next))); + sourceSha = state.resultSha; + if (!sourceSha.equals(gitea.head(remote.repository()))) + throw new ImportFailure("README_MAIN_CHANGED", "README 处理后的 main 已变化"); + } project.setGiteaRepoId(remote.id()); project.setRepoStatus("READY"); @@ -271,6 +294,8 @@ public final class ImportCoordinator { } catch (RuntimeException warningFailure) { warnings = List.of("兼容性扫描未完成;请人工检查大文件、LFS、子模块和符号链接"); } + warnings = new ArrayList<>(warnings); + if (entry.readmeImages != null) warnings.addAll(entry.readmeImages.warnings); List checkedWarnings = warnings; store.update(entry, e -> e.warnings = new ArrayList<>(checkedWarnings)); store.phase(entry, Phase.PUBLISHING); @@ -388,4 +413,9 @@ public final class ImportCoordinator { WorkbookReader.repositoryKey(entry.candidate.repositoryUrl())))) throw new ImportFailure("REPOSITORY_CONFLICT", "目标仓库归属、来源、可见性或内容不符合迁移要求"); } + private void checkpointReadme(Entry entry, com.linkerbot.vibemake.service.readme.ReadmeRepository.State state) { + try { store.update(entry, current -> { current.readmeImages = state; current.phase = Phase.README_IMAGES; }); } + catch (IOException error) { throw new java.io.UncheckedIOException(error); } + } + } diff --git a/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportModels.java b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportModels.java index fa696ac..c7bf57b 100644 --- a/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportModels.java +++ b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/ImportModels.java @@ -39,6 +39,7 @@ public final class ImportModels { MIGRATED, NORMALIZING, BRANCH_READY, + README_IMAGES, LINKED, PUBLISHING, PUBLISHED, @@ -55,6 +56,7 @@ public final class ImportModels { public String reservedAt; public RemoteRepository receipt; public String targetSha; + public com.linkerbot.vibemake.service.readme.ReadmeRepository.State readmeImages; public String errorCode; public String failedPhase; public String message; diff --git a/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/cover/CoverOssUploader.java b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/cover/CoverOssUploader.java index fc539fb..9be97b2 100644 --- a/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/cover/CoverOssUploader.java +++ b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/cover/CoverOssUploader.java @@ -1,72 +1,19 @@ package com.linkerbot.vibemake.importer.cover; -import com.aliyun.oss.ClientBuilderConfiguration; -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSClientBuilder; -import com.aliyun.oss.OSSException; -import com.aliyun.oss.common.auth.DefaultCredentialProvider; -import com.aliyun.oss.common.comm.SignVersion; -import com.aliyun.oss.model.ObjectMetadata; -import com.aliyun.oss.model.PutObjectRequest; import com.linkerbot.vibemake.service.config.OssProperties; -import java.io.ByteArrayInputStream; -import java.util.Map; +import com.linkerbot.vibemake.service.readme.ExternalImageDownloader; +import com.linkerbot.vibemake.service.readme.ImageOssUploader; +/** 封面和 README 复用同一套 OSS 摘要核验上传。 */ final class CoverOssUploader implements AutoCloseable, CoverMigrator.Uploader { - private final OSS client; - private final String bucket; - - CoverOssUploader(OssProperties properties) { - properties.setEnabled(true); - if (!properties.isConfigurationValid()) throw new ExternalCoverDownloader.Failure("OSS_CONFIGURATION_MISSING"); - ClientBuilderConfiguration configuration = new ClientBuilderConfiguration(); - configuration.setSignatureVersion(SignVersion.V4); - configuration.setConnectionTimeout(10000); - configuration.setSocketTimeout(20000); - configuration.setRequestTimeoutEnabled(true); - configuration.setRequestTimeout(30000); - configuration.setMaxErrorRetry(1); - client = OSSClientBuilder.create().endpoint(properties.resolvedEndpoint()).region(properties.getRegion()) - .credentialsProvider(new DefaultCredentialProvider(properties.getAccessKeyId(), properties.getAccessKeySecret())) - .clientConfiguration(configuration).build(); - bucket = properties.getBucket(); - } - - @Override - public void upload(String key, ExternalCoverDownloader.Image image, String sha256) { - ObjectMetadata existing = metadata(key); - if (existing != null) { - verify(existing, image, sha256); - return; - } - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(image.bytes().length); - metadata.setContentType(image.contentType()); - metadata.setUserMetadata(Map.of("sha256", sha256)); - metadata.setHeader("x-oss-forbid-overwrite", "true"); + private final ImageOssUploader uploader; + CoverOssUploader(OssProperties properties) { uploader = new ImageOssUploader(properties); } + @Override public void upload(String key, ExternalCoverDownloader.Image image, String sha256) { try { - client.putObject(new PutObjectRequest(bucket, key, new ByteArrayInputStream(image.bytes()), metadata)); - } catch (OSSException error) { - if (!"FileAlreadyExists".equals(error.getErrorCode())) - throw new ExternalCoverDownloader.Failure("OSS_PUT_" + error.getErrorCode()); - } - ObjectMetadata uploaded = metadata(key); - if (uploaded == null) throw new ExternalCoverDownloader.Failure("OSS_VERIFY_MISSING"); - verify(uploaded, image, sha256); - } - - private ObjectMetadata metadata(String key) { - try { return client.getObjectMetadata(bucket, key); } - catch (OSSException error) { - if ("NoSuchKey".equals(error.getErrorCode())) return null; - throw new ExternalCoverDownloader.Failure("OSS_HEAD_" + error.getErrorCode()); + uploader.upload(key, new ExternalImageDownloader.Image(image.bytes(), image.contentType(), image.extension()), sha256); + } catch (ExternalImageDownloader.Failure error) { + throw new ExternalCoverDownloader.Failure(error.code()); } } - - private void verify(ObjectMetadata metadata, ExternalCoverDownloader.Image image, String sha256) { - if (metadata.getContentLength() != image.bytes().length || !sha256.equals(metadata.getUserMetadata().get("sha256"))) - throw new ExternalCoverDownloader.Failure("OSS_VERIFY_MISMATCH"); - } - - @Override public void close() { client.shutdown(); } + @Override public void close() { uploader.close(); } } diff --git a/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/readme/ReadmeMigrationApplication.java b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/readme/ReadmeMigrationApplication.java new file mode 100644 index 0000000..fb120d5 --- /dev/null +++ b/vibemake-import/src/main/java/com/linkerbot/vibemake/importer/readme/ReadmeMigrationApplication.java @@ -0,0 +1,296 @@ +package com.linkerbot.vibemake.importer.readme; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkerbot.vibemake.bean.entity.Project; +import com.linkerbot.vibemake.common.BusinessException; +import com.linkerbot.vibemake.service.config.*; +import com.linkerbot.vibemake.service.gitea.*; +import com.linkerbot.vibemake.service.mapper.ProjectMapper; +import com.linkerbot.vibemake.service.oss.OssResourcePaths; +import com.linkerbot.vibemake.service.readme.*; +import com.linkerbot.vibemake.service.support.*; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.*; +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.jdbc.core.JdbcTemplate; + +/** 存量 README 运维入口;备份与逐阶段收据先于远端写入持久化。 */ +public final class ReadmeMigrationApplication { + @Configuration(proxyBeanMethods = false) + @EnableAutoConfiguration + @MapperScan("com.linkerbot.vibemake.service.mapper") + @Import({MybatisConfiguration.class, VibemakeProperties.class, OssProperties.class, OssResourcePaths.class, + ReadmeImageProperties.class, RestGiteaClient.class, ProjectLocks.class, + ImageOssUploader.class, ReadmeImages.class, ReadmeRepository.class}) + public static class Config {} + + public record Options(boolean execute, int limit, Set codes, Path directory) { + public static Options parse(String[] args) { + boolean execute = false, dryRun = false; + int limit = Integer.MAX_VALUE; + Path directory = Path.of("logs/readme-image-migration"); + Set codes = new LinkedHashSet<>(); + for (int index = 0; index < args.length; index++) { + String argument = args[index]; + if (argument.equals("--execute")) execute = true; + else if (argument.equals("--dry-run")) dryRun = true; + else { + if (++index == args.length) throw new IllegalArgumentException("缺少参数值"); + switch (argument) { + case "--limit" -> limit = Integer.parseInt(args[index]); + case "--project-code" -> codes.add(args[index]); + case "--state-dir" -> directory = Path.of(args[index]); + default -> throw new IllegalArgumentException("不支持的参数"); + } + } + } + if (execute && dryRun || limit < 1 || codes.stream().anyMatch(code -> !code.matches("[A-Za-z0-9_-]{1,64}"))) + throw new IllegalArgumentException("参数不合法"); + return new Options(execute, limit, Set.copyOf(codes), directory); + } + } + + public static class Job { + public String code; + public String runCode; + public Long repositoryId; + public String owner; + public String name; + public String originalMain; + public String originalPublished; + public String publishStatus; + public ReadmeRepository.State main; + public ReadmeRepository.State published; + public String status; + public String errorCode; + } + + private final Options options; + private final ObjectMapper json; + private final JdbcTemplate jdbc; + private final ProjectMapper projects; + private final GiteaClient gitea; + private final ProjectLocks locks; + private final ReadmeRepository readmes; + + public ReadmeMigrationApplication(Options options, ConfigurableApplicationContext context) { + this.options = options; + json = context.getBean(ObjectMapper.class); + jdbc = context.getBean(JdbcTemplate.class); + projects = context.getBean(ProjectMapper.class); + gitea = context.getBean(GiteaClient.class); + locks = context.getBean(ProjectLocks.class); + readmes = context.getBean(ReadmeRepository.class); + } + + public static int runCli(String[] args) { + if (Arrays.asList(args).contains("--help")) { + System.out.println("用法:readme-images [--dry-run | --execute] [--limit N] [--project-code 编码] [--state-dir logs/readme-image-migration]"); + return 0; + } + try { + Options options = Options.parse(args); + try (ConfigurableApplicationContext context = new SpringApplicationBuilder(Config.class) + .web(WebApplicationType.NONE).logStartupInfo(false).run()) { + return new ReadmeMigrationApplication(options, context).run(context.getBean(VibemakeProperties.class), context.getBean(OssProperties.class)); + } + } catch (Exception error) { + System.err.println("README 迁移终止 code=" + code(error)); + return 1; + } + } + + int run(VibemakeProperties config, OssProperties oss) throws IOException { + Files.createDirectories(options.directory()); + restrict(options.directory(), "rwx------"); + try (FileChannel channel = FileChannel.open(options.directory().resolve(".lock"), StandardOpenOption.CREATE, StandardOpenOption.WRITE); + FileLock ignored = channel.tryLock()) { + if (ignored == null) throw new IllegalStateException("状态目录正在使用"); + restrict(options.directory().resolve(".lock"), "rw-------"); + Map binding = Map.of("gitea", config.getGitea().getBaseUrl(), "organization", config.getGitea().getOwner(), + "bucket", oss.getBucket(), "root", oss.getRootDirectory(), "publicBase", oss.publicBaseUrl()); + Path manifest = options.directory().resolve("environment.json"); + if (Files.exists(manifest) && !json.readTree(Files.readString(manifest)).equals(json.valueToTree(binding))) + throw new IllegalStateException("状态目录绑定的环境已变化"); + write(manifest, json.writeValueAsBytes(binding)); + List candidates = projects.selectList(new LambdaQueryWrapper().eq(Project::getDeleted, false).orderByAsc(Project::getProjectCode)); + List> report = new ArrayList<>(); + int processed = 0; + for (Project project : candidates) { + String code = project.getProjectCode(); + if (!options.codes().isEmpty() && !options.codes().contains(code)) continue; + if (!"READY".equals(project.getRepoStatus())) { + report.add(Map.of("projectCode", code, "status", "SKIPPED", "reason", "REPOSITORY_NOT_READY")); + continue; + } + if (processed >= options.limit()) break; + try { + Map result = process(project); + report.add(result); + if (Boolean.TRUE.equals(result.get("candidate")) && !"ALREADY_DONE".equals(result.get("status"))) processed++; + System.out.printf("README projectCode=%s status=%s candidates=%s replaced=%s failed=%s%n", code, + result.get("status"), result.getOrDefault("images", 0), result.getOrDefault("replaced", 0), result.getOrDefault("failed", 0)); + } catch (RuntimeException error) { + report.add(Map.of("projectCode", code, "status", "FAILED", "errorCode", code(error))); + System.err.printf("README projectCode=%s status=FAILED code=%s%n", code, code(error)); + } + write(options.directory().resolve(options.execute() ? "report.json" : "preview.json"), json.writerWithDefaultPrettyPrinter().writeValueAsBytes(report)); + } + write(options.directory().resolve(options.execute() ? "report.json" : "preview.json"), json.writerWithDefaultPrettyPrinter().writeValueAsBytes(report)); + System.out.printf("README 批次结束 projects=%d candidates=%d reportDirectory=%s%n", report.size(), processed, options.directory().toAbsolutePath()); + return report.stream().anyMatch(item -> "FAILED".equals(item.get("status"))) ? 2 : 0; + } + } + + private Map process(Project project) { + String code = project.getProjectCode(); + GiteaClient.Repository repository = new GiteaClient.Repository(project.getGiteaRepoId(), project.getGiteaOwner(), project.getGiteaRepoName()); + Path stateFile = options.directory().resolve(code + ".json"); + Job job = options.execute() && Files.exists(stateFile) ? read(stateFile, Job.class) : null; + if (job != null && "DONE".equals(job.status) && "IDLE".equals(project.getRepoWriteStatus())) + return result(job, "ALREADY_DONE"); + if (job == null) { + job = new Job(); + job.code = code; + job.runCode = UUID.randomUUID().toString().substring(0, 8); + job.repositoryId = repository.id(); job.owner = repository.owner(); job.name = repository.name(); + job.originalMain = gitea.head(repository); + job.originalPublished = project.getPublishedCommitSha(); + job.publishStatus = project.getPublishStatus(); + } + Job current = job; + if (!Objects.equals(job.repositoryId, project.getGiteaRepoId()) || !job.owner.equals(project.getGiteaOwner()) || !job.name.equals(project.getGiteaRepoName())) + throw BusinessException.conflict("仓库身份已变化"); + GiteaClient.RemoteRepository remote = gitea.inspect(repository.name()); + if (remote.id() != repository.id() || !remote.owner().equals(repository.owner()) || !remote.privateRepo() || remote.mirror() || remote.empty()) + throw BusinessException.conflict("远端仓库身份已变化"); + boolean same = Objects.equals(job.originalMain, job.originalPublished); + ReadmeRepository.Prepared main = job.main == null ? readmes.prepare(code, repository, job.originalMain, false) : null; + ReadmeRepository.Prepared published = job.originalPublished == null || same || job.published != null ? null + : readmes.prepare(code, repository, job.originalPublished, false); + if (main != null && main.path() == null && main.result().failed() > 0 + || published != null && published.path() == null && published.result().failed() > 0) + throw new ExternalImageDownloader.Failure("README_SCAN_FAILED"); + int images = (main == null ? 0 : main.scan().images().size()) + (published == null ? 0 : published.scan().images().size()); + int badges = (main == null ? 0 : main.scan().badges()) + (published == null ? 0 : published.scan().badges()); + if (!options.execute() || images == 0 && job.main == null && job.published == null) + return Map.of("projectCode", code, "candidate", images > 0, "status", images > 0 ? "PREVIEW" : "SKIPPED", + "images", images, "badges", badges, "publishedDifferent", job.originalPublished != null && !same); + save(stateFile, job); + backup(code + "-main-original.md", main); + backup(code + "-published-original.md", published); + // 所有外网图片下载和上传均在项目锁之外。 + ReadmeRepository.Prepared mainReady = main == null ? null : readmes.prepare(code, repository, job.originalMain, true); + ReadmeRepository.Prepared publishedReady = published == null ? null : readmes.prepare(code, repository, job.originalPublished, true); + try { + locks.serial(code, () -> { + Project fresh = projects.selectOne(new LambdaQueryWrapper().eq(Project::getProjectCode, code)); + if (fresh == null || Boolean.TRUE.equals(fresh.getDeleted()) || !"READY".equals(fresh.getRepoStatus()) + || !Objects.equals(fresh.getGiteaRepoId(), current.repositoryId) || !fresh.getGiteaOwner().equals(current.owner) + || !fresh.getGiteaRepoName().equals(current.name) || !fresh.getPublishStatus().equals(current.publishStatus)) + throw BusinessException.conflict("项目状态已变化"); + Long active = jdbc.queryForObject("SELECT COUNT(*) FROM repository_write_operation WHERE project_code = ? AND status IN ('PROCESSING','NEEDS_REVIEW')", Long.class, code); + if (active != null && active > 0) throw BusinessException.conflict("项目存在待核对操作"); + if (!"IDLE".equals(fresh.getRepoWriteStatus())) { + if (!"REVIEW".equals(fresh.getRepoWriteStatus()) || current.main == null && current.published == null) + throw BusinessException.conflict("项目存在其他待核对写入"); + if (current.main != null) readmes.resume(code, repository, current.main, state -> { current.main = state; save(stateFile, current); }); + if (current.published != null && !same) readmes.resume(code, repository, current.published, state -> { current.published = state; save(stateFile, current); }); + locks.confirmKnownResult(code); + } + String expectedMain = current.main == null || current.main.resultSha == null ? current.originalMain : current.main.resultSha; + if (!expectedMain.equals(gitea.head(repository))) throw BusinessException.conflict("main 已变化,跳过本次处理"); + String expectedPublished = same && current.main != null ? current.main.resultSha : current.published == null || current.published.resultSha == null ? current.originalPublished : current.published.resultSha; + if (!Objects.equals(fresh.getPublishedCommitSha(), current.originalPublished) && !Objects.equals(fresh.getPublishedCommitSha(), expectedPublished)) + throw BusinessException.conflict("发布版本已变化,跳过本次处理"); + return locks.write(code, () -> { + current.main = readmes.apply(code, repository, current.originalMain, "main", mainReady, current.main, + state -> { current.main = state; save(stateFile, current); }); + if (same) current.published = current.main; + else if (current.originalPublished != null) + current.published = readmes.apply(code, repository, current.originalPublished, + "maintenance/readme-oss/" + code + "/" + current.runCode, publishedReady, current.published, + state -> { current.published = state; save(stateFile, current); }); + if (current.published != null && !current.originalPublished.equals(current.published.resultSha) + && !Objects.equals(fresh.getPublishedCommitSha(), current.published.resultSha)) { + int changed = jdbc.update("UPDATE project SET published_commit_sha = ?, updated_at = ? WHERE project_code = ? AND deleted = 0" + + " AND repo_status = 'READY' AND publish_status = ? AND HEX(published_commit_sha) = HEX(?) AND gitea_repo_id = ?", + current.published.resultSha, ProjectSupport.now(), code, current.publishStatus, current.originalPublished, current.repositoryId); + if (changed != 1) throw BusinessException.conflict("发布引用已变化"); + } + current.status = "DONE"; + save(stateFile, current); + return null; + }); + }); + } catch (RuntimeException error) { + current.status = "FAILED"; + current.errorCode = code(error); + save(stateFile, current); + throw error; + } + return result(current, "DONE"); + } + + private Map result(Job job, String status) { + boolean same = Objects.equals(job.originalMain, job.originalPublished); + int replaced = job.main.replaced + (same || job.published == null ? 0 : job.published.replaced); + int failed = job.main.failed + (same || job.published == null ? 0 : job.published.failed); + int badges = job.main.badges + (same || job.published == null ? 0 : job.published.badges); + return Map.of("projectCode", job.code, "candidate", true, "status", status, "images", replaced + failed, + "replaced", replaced, "failed", failed, "badges", badges, + "mainSha", job.main.resultSha, "publishedSha", job.published == null ? "" : job.published.resultSha); + } + + private void backup(String name, ReadmeRepository.Prepared prepared) { + if (prepared == null || prepared.path() == null) return; + Path path = options.directory().resolve(name); + if (!Files.exists(path)) writeUnchecked(path, prepared.original().getBytes(StandardCharsets.UTF_8)); + } + private T read(Path path, Class type) { + try { return json.readValue(Files.readAllBytes(path), type); } + catch (IOException error) { throw new UncheckedIOException(error); } + } + private void save(Path path, Object value) { + try { write(path, json.writerWithDefaultPrettyPrinter().writeValueAsBytes(value)); } + catch (IOException error) { throw new UncheckedIOException(error); } + } + private static void writeUnchecked(Path path, byte[] bytes) { + try { write(path, bytes); } catch (IOException error) { throw new UncheckedIOException(error); } + } + private static void write(Path path, byte[] bytes) throws IOException { + Path temporary = Files.createTempFile(path.getParent(), ".readme-", ".tmp"); + try { + restrict(temporary, "rw-------"); + try (FileChannel output = FileChannel.open(temporary, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + ByteBuffer buffer = ByteBuffer.wrap(bytes); + while (buffer.hasRemaining()) output.write(buffer); + output.force(true); + } + Files.move(temporary, path, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + try (FileChannel directory = FileChannel.open(path.getParent(), StandardOpenOption.READ)) { directory.force(true); } + } finally { Files.deleteIfExists(temporary); } + } + private static void restrict(Path path, String mode) throws IOException { + if (Files.getFileStore(path).supportsFileAttributeView("posix")) Files.setPosixFilePermissions(path, PosixFilePermissions.fromString(mode)); + } + private static String code(Throwable error) { + if (error instanceof BusinessException business) return business.code(); + if (error instanceof ExternalImageDownloader.Failure failure) return failure.code(); + return error.getClass().getSimpleName(); + } +} diff --git a/vibemake-import/src/main/resources/application.yml b/vibemake-import/src/main/resources/application.yml index 4eba35a..639f3c0 100644 --- a/vibemake-import/src/main/resources/application.yml +++ b/vibemake-import/src/main/resources/application.yml @@ -17,8 +17,12 @@ mybatis-plus: configuration: map-underscore-to-camel-case: true vibemake: + readme-images: + enabled: ${README_IMAGES_ENABLED:true} + max-bytes: ${README_IMAGE_MAX_BYTES:20971520} + concurrency: ${README_IMAGE_CONCURRENCY:2} oss: - # 导入只转换资源地址,永不启用 OSS SDK 或校验上传凭据。 + # 常规资源仅转换地址;README 转存按需单独校验凭据,预览不初始化 OSS SDK。 enabled: false region: ${OSS_REGION:cn-beijing} bucket: ${OSS_BUCKET:linkerbot} diff --git a/vibemake-import/src/test/java/com/linkerbot/vibemake/importer/ImportCoordinatorTest.java b/vibemake-import/src/test/java/com/linkerbot/vibemake/importer/ImportCoordinatorTest.java index 6fc6e1e..5656e22 100644 --- a/vibemake-import/src/test/java/com/linkerbot/vibemake/importer/ImportCoordinatorTest.java +++ b/vibemake-import/src/test/java/com/linkerbot/vibemake/importer/ImportCoordinatorTest.java @@ -18,6 +18,8 @@ import com.linkerbot.vibemake.service.oss.OssObjectStore; import com.linkerbot.vibemake.service.oss.OssResourcePaths; import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.junit.jupiter.api.io.TempDir; import org.mockito.AdditionalAnswers; import org.springframework.beans.factory.annotation.Autowired; @@ -178,6 +180,39 @@ class ImportCoordinatorTest { Duration.ofMinutes(30)); } + @ParameterizedTest + @ValueSource(strings = {"master", "main"}) + void readmeCommitIsPublishedWhileOriginalMigrationShaRemainsFrozen(String sourceBranch) throws Exception { + if (sourceBranch.equals("main")) { + entry.source = new GitHubRepository(entry.source.id(), entry.source.url(), "main"); + main.set(true); + remote = new GiteaClient.RemoteRepository(remote.id(), remote.owner(), remote.name(), true, false, false, remote.originalUrl(), "main"); + store.save(); + } + com.linkerbot.vibemake.service.readme.ReadmeRepository readmes = mock(com.linkerbot.vibemake.service.readme.ReadmeRepository.class); + String imageSha = "b".repeat(40); + when(readmes.apply(anyString(), any(), eq(SHA), eq("main"), any(), isNull(), any())).thenAnswer(call -> { + com.linkerbot.vibemake.service.readme.ReadmeRepository.State state = new com.linkerbot.vibemake.service.readme.ReadmeRepository.State(); + state.baseSha = SHA; state.resultSha = imageSha; state.branch = "main"; state.stage = "DONE"; + com.linkerbot.vibemake.service.readme.ReadmeRepository.Checkpoint checkpoint = call.getArgument(6); + checkpoint.save(state); + when(gitea.head(any())).thenReturn(imageSha); + when(gitea.branch(any(), eq("main"))).thenReturn(imageSha); + return state; + }); + worker(projects, service).readmes(readmes).run(entry); + assertThat(entry.phase).isEqualTo(Phase.PUBLISHED); + assertThat(entry.targetSha).isEqualTo(SHA); + assertThat(entry.readmeImages.resultSha).isEqualTo(imageSha); + assertThat(service.published(entry.projectCode).getPublishedCommitSha()).isEqualTo(imageSha); + // 模拟发布已成功、批次终态尚未落盘时重启。 + entry.phase = Phase.PUBLISHING; + store.save(); + worker(projects, service).readmes(readmes).run(entry); + assertThat(entry.phase).isEqualTo(Phase.PUBLISHED); + verify(gitea, times(1)).migrate(anyString(), anyString(), any(Duration.class)); + } + @Test void migrationBindsShortCodeAdaptsMasterAndPublishesWithoutEmptyRepository() throws Exception { Instant before = Instant.now().truncatedTo(java.time.temporal.ChronoUnit.MILLIS); diff --git a/vibemake-service/pom.xml b/vibemake-service/pom.xml index b1cc77c..e2b3473 100644 --- a/vibemake-service/pom.xml +++ b/vibemake-service/pom.xml @@ -8,6 +8,11 @@ vibemake-service + + com.vladsch.flexmark + flexmark + 0.64.8 + org.springframework.boot spring-boot-starter-data-redis diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/admin/AdminImportService.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/admin/AdminImportService.java index 7ee7589..c987b63 100644 --- a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/admin/AdminImportService.java +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/admin/AdminImportService.java @@ -17,6 +17,7 @@ import com.linkerbot.vibemake.service.migration.*; import com.linkerbot.vibemake.service.oss.OssResourcePaths; import com.linkerbot.vibemake.service.oss.UrdfResources; import com.linkerbot.vibemake.service.support.*; +import com.linkerbot.vibemake.service.readme.*; import lombok.RequiredArgsConstructor; @@ -47,6 +48,7 @@ public class AdminImportService { private final UrdfResources urdfResources; private final ObjectMapper json; private final TransactionTemplate transactions; + private final ReadmeRepository readmes; public ImportPreview preview(String url) { management.requireAdmin(); @@ -306,12 +308,15 @@ public class AdminImportService { } else if ("REVIEW".equals(task.getStatus())) { if (task.getReceiptJson() == null) throw BusinessException.conflict("迁移结果不明,需要运维核对,禁止重发"); - if ("MIGRATED".equals(task.getPhase())) { + if (readmeState(task) != null) { + GiteaClient.RemoteRepository remote = engine().verifyReceipt(task.getSourceRepositoryUrl(), project.getGiteaRepoName(), receipt(task)); + readmes.resume(task.getProjectCode(), remote.repository(), readmeState(task), state -> saveReadme(task, remote, state)); + } else if ("MIGRATED".equals(task.getPhase())) { engine().verifyReceipt( task.getSourceRepositoryUrl(), project.getGiteaRepoName(), receipt(task)); - } else if (Set.of("NORMALIZING", "BRANCH_READY").contains(task.getPhase()) + } else if (Set.of("NORMALIZING", "BRANCH_READY", "README_IMAGES").contains(task.getPhase()) && task.getTargetSha() != null) { engine().verifyNormalized( task.getSourceRepositoryUrl(), @@ -367,8 +372,8 @@ public class AdminImportService { if ("CLEANING".equals(task.getPhase())) return false; if ("FAILED".equals(task.getStatus())) return true; return task.getReceiptJson() != null - && ("MIGRATED".equals(task.getPhase()) - || Set.of("NORMALIZING", "BRANCH_READY").contains(task.getPhase()) + && (readmeState(task) != null || "MIGRATED".equals(task.getPhase()) + || Set.of("NORMALIZING", "BRANCH_READY", "README_IMAGES").contains(task.getPhase()) && task.getTargetSha() != null); } @@ -658,7 +663,10 @@ public class AdminImportService { recoverMissingReceipt(task, project); } RepositoryMigration.Result result; - if ("BRANCH_READY".equals(task.getPhase()) && task.getReceiptJson() != null) { + if (readmeState(task) != null) { + GiteaClient.RemoteRepository remote = engine().verifyReceipt(task.getSourceRepositoryUrl(), project.getGiteaRepoName(), receipt(task)); + result = new RepositoryMigration.Result(remote, task.getTargetSha()); + } else if ("BRANCH_READY".equals(task.getPhase()) && task.getReceiptJson() != null) { result = engine().verifyNormalized( task.getSourceRepositoryUrl(), @@ -729,16 +737,20 @@ public class AdminImportService { } }); } + ReadmeRepository.State imageState = migrateReadme(task, result); + String finalSha = imageState.resultSha; List warnings; try { warnings = gitea.migrationWarnings( result.remote().repository(), - result.sha(), + finalSha, properties.getMaxFileBytes()); } catch (RuntimeException error) { warnings = List.of("兼容性扫描未完成,请检查大文件、LFS、子模块及符号链接"); } + warnings = new ArrayList<>(warnings); + warnings.addAll(imageState.warnings); String warningsJson = json.writeValueAsString(warnings); transactions.executeWithoutResult( status -> { @@ -771,7 +783,7 @@ public class AdminImportService { task.getOperatorUserCode(), task.getTaskCode(), task.getProjectCode(), - result.sha()); + finalSha); } catch (IOException error) { throw new IllegalStateException("迁移收据持久化未确认", error); } @@ -782,7 +794,7 @@ public class AdminImportService { try { ProjectImportTask task = find(code); boolean review = - Set.of("CLEANING", "MIGRATING", "VERIFYING", "MIGRATED", "NORMALIZING", "BRANCH_READY") + Set.of("CLEANING", "MIGRATING", "VERIFYING", "MIGRATED", "NORMALIZING", "BRANCH_READY", "README_IMAGES") .contains(task.getPhase()); if (error instanceof GiteaFailure failure && failure.uncertain()) review = true; if (error instanceof MigrationFailure failure @@ -844,7 +856,7 @@ public class AdminImportService { status -> { task.setPhase(phase); save(task); - if (Set.of("CLEANING", "MIGRATING", "NORMALIZING").contains(phase)) { + if (Set.of("CLEANING", "MIGRATING", "NORMALIZING", "README_IMAGES").contains(phase)) { int updated = projects.update( null, @@ -865,7 +877,7 @@ public class AdminImportService { private GiteaClient.RemoteRepository receipt(ProjectImportTask task) { if (task.getReceiptJson() == null) return null; try { - return json.readValue(task.getReceiptJson(), GiteaClient.RemoteRepository.class); + return ReadmeMigrationReceipt.repository(json, task.getReceiptJson()); } catch (IOException error) { throw new IllegalStateException("迁移收据无效"); } @@ -874,4 +886,28 @@ public class AdminImportService { private RepositoryMigration engine() { return new RepositoryMigration(gitea, properties.getGitea().getOwner()); } + private ReadmeRepository.State readmeState(ProjectImportTask task) { + try { return ReadmeMigrationReceipt.state(json, task.getReceiptJson()); } + catch (IOException error) { throw new IllegalStateException("README 收据无效"); } + } + + private void saveReadme(ProjectImportTask task, GiteaClient.RemoteRepository remote, ReadmeRepository.State state) { + try { task.setReceiptJson(ReadmeMigrationReceipt.encode(json, remote, state)); } + catch (IOException error) { throw new IllegalStateException("README 收据编码失败"); } + task.setPhase("README_IMAGES"); + save(task); + } + + private ReadmeRepository.State migrateReadme(ProjectImportTask task, RepositoryMigration.Result result) { + ReadmeRepository.State previous = readmeState(task); + phase(task, "README_IMAGES"); + ReadmeRepository.Prepared prepared = previous == null + ? readmes.prepare(task.getProjectCode(), result.remote().repository(), result.sha(), true) : null; + ReadmeRepository.State state = readmes.apply(task.getProjectCode(), result.remote().repository(), result.sha(), "main", prepared, previous, + next -> saveReadme(task, result.remote(), next)); + if (!state.resultSha.equals(gitea.head(result.remote().repository()))) + throw new MigrationFailure("README_MAIN_CHANGED", "README 处理后的 main 版本已变化"); + return state; + } + } diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/config/ReadmeImageProperties.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/config/ReadmeImageProperties.java new file mode 100644 index 0000000..fbcce28 --- /dev/null +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/config/ReadmeImageProperties.java @@ -0,0 +1,37 @@ +package com.linkerbot.vibemake.service.config; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; +import org.springframework.validation.annotation.Validated; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.Max; + +@Getter +@Setter +@Component +@Validated +@ConfigurationProperties(prefix = "vibemake.readme-images") +public class ReadmeImageProperties { + private boolean enabled = true; + @Min(1) @Max(50 * 1024 * 1024) + private int maxBytes = 20 * 1024 * 1024; + @Min(1) @Max(4) + private int concurrency = 2; + /** 完整 URL 的正则表达式;只用于配置,不记录匹配到的原始地址。 */ + private List preserveUrlPatterns = new ArrayList<>(); + + @jakarta.validation.constraints.AssertTrue(message = "README 保留地址规则必须是合法的正则表达式") + public boolean isPreserveUrlPatternsValid() { + if (preserveUrlPatterns == null) return false; + try { + for (String expression : preserveUrlPatterns) java.util.regex.Pattern.compile(expression); + return true; + } catch (IllegalArgumentException | NullPointerException error) { + return false; + } + } +} diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java index 22a8358..8a375af 100644 --- a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/GiteaClient.java @@ -45,6 +45,12 @@ public interface GiteaClient { throw new UnsupportedOperationException("未实现仓库写入凭据核对"); } + default String findWriteReceipt(Repository repository, String branch, String baseSha, + String operationCode, String requestDigest) { + if ("main".equals(branch)) return findWriteReceipt(repository, baseSha, operationCode, requestDigest); + throw new UnsupportedOperationException("未实现分支写入凭据核对"); + } + record RemoteRepository(long id, String owner, String name, boolean privateRepo, boolean mirror, boolean empty, String originalUrl, String defaultBranch) { public Repository repository() { return new Repository(id, owner, name); } @@ -99,5 +105,11 @@ public interface GiteaClient { String authorName, String authorCode); + default CommitView commit(Repository repository, String branch, List changes, + String message, String authorName, String authorCode) { + if ("main".equals(branch)) return commit(repository, changes, message, authorName, authorCode); + throw new UnsupportedOperationException("未实现指定分支提交"); + } + PageResponse history(Repository repository, long page, long size); } diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java index 96992ca..34815ea 100644 --- a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/gitea/RestGiteaClient.java @@ -490,6 +490,12 @@ public class RestGiteaClient implements GiteaClient { String message, String authorName, String authorCode) { + return commit(repo, "main", changes, message, authorName, authorCode); + } + + @Override + public CommitView commit(Repository repo, String branch, List changes, + String message, String authorName, String authorCode) { List> files = new ArrayList<>(); for (Change change : changes) { Map item = new LinkedHashMap<>(); @@ -503,7 +509,7 @@ public class RestGiteaClient implements GiteaClient { Map body = Map.of( "branch", - "main", + branch, "files", files, "message", @@ -550,10 +556,16 @@ public class RestGiteaClient implements GiteaClient { @Override public String findWriteReceipt(Repository repo, String baseSha, String operationCode, String requestDigest) { + return findWriteReceipt(repo, "main", baseSha, operationCode, requestDigest); + } + + @Override + public String findWriteReceipt(Repository repo, String branch, String baseSha, String operationCode, + String requestDigest) { // 普通文件写入被项目锁串行化,未知期间禁止下一次写入。限制扫描量防止异常仓库拖垮任务。 // 超出扫描范围不推断失败,留在处理中并交给管理员兜底。 String suffix = WriteReceipt.suffix(operationCode, requestDigest); - String head = head(repo); + String head = branch(repo, branch); String found = null; for (int page = 1; page <= 20; page++) { JsonNode entries = parse(request("GET", repoPath(repo) @@ -573,7 +585,7 @@ public class RestGiteaClient implements GiteaClient { } if (entries.size() < 50) break; } - if (!head.equals(head(repo))) + if (!head.equals(branch(repo, branch))) throw new GiteaFailure(409, "WRITE_RECEIPT_BRANCH_CHANGED", "核对期间分支版本已变化", false); return found; } diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/oss/OssResourcePaths.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/oss/OssResourcePaths.java index 466338d..bc8275e 100644 --- a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/oss/OssResourcePaths.java +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/oss/OssResourcePaths.java @@ -66,7 +66,8 @@ public class OssResourcePaths { if (root != null && !root.isBlank() && path.startsWith(root + "/")) path = path.substring(root.length()); if (!path.startsWith("/project-covers/") && !path.startsWith("/urdf/") - && !path.startsWith("/project-resources/")) throw invalidPath(); + && !path.startsWith("/project-resources/") + && !path.startsWith("/readme-images/")) throw invalidPath(); String relative = path.substring(1); if (relative.endsWith("/")) relative = relative.substring(0, relative.length() - 1); try { diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ExternalImageDownloader.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ExternalImageDownloader.java new file mode 100644 index 0000000..3773022 --- /dev/null +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ExternalImageDownloader.java @@ -0,0 +1,201 @@ +package com.linkerbot.vibemake.service.readme; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; +import java.util.Locale; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.NodeList; +import org.xml.sax.helpers.DefaultHandler; + +/** 公开图片下载:DNS 校验与实际连接共用解析结果,不附加任何业务凭据。 */ +public class ExternalImageDownloader { + private static final java.util.concurrent.ScheduledExecutorService TIMEOUTS = + java.util.concurrent.Executors.newSingleThreadScheduledExecutor(task -> { + Thread thread = new Thread(task, "readme-image-timeouts"); + thread.setDaemon(true); + return thread; + }); + public record Image(byte[] bytes, String contentType, String extension) {} + public static class Failure extends RuntimeException { + public Failure(String code) { super(code); } + public String code() { return getMessage(); } + } + + private final int maxBytes; + public ExternalImageDownloader(int maxBytes) { + if (maxBytes < 1 || maxBytes > 50 * 1024 * 1024) throw new IllegalArgumentException("图片大小限制不合法"); + this.maxBytes = maxBytes; + } + + public Image download(String url) throws Exception { + long deadline = System.nanoTime() + Duration.ofSeconds(45).toNanos(); + for (int attempt = 0; ; attempt++) { + try { return fetch(url, deadline); } + catch (IOException error) { + if (System.nanoTime() >= deadline) throw new Failure("IMAGE_DOWNLOAD_TIMEOUT"); + if (attempt >= 2 || Thread.currentThread().isInterrupted()) throw error; + } + } + } + + private Image fetch(String url, long deadline) throws Exception { + URI uri = URI.create(url.startsWith("//") ? "https:" + url : url); + try (CloseableHttpClient client = HttpClients.custom().disableAutomaticRetries().disableRedirectHandling().disableCookieManagement() + .setDnsResolver(host -> { + InetAddress[] addresses = InetAddress.getAllByName(host); + for (InetAddress address : addresses) validateAddress(address); + return addresses; + }).build()) { + for (int redirects = 0; redirects <= 5; redirects++) { + validateUri(uri); + int remaining = (int) Math.min(Integer.MAX_VALUE, (deadline - System.nanoTime()) / 1_000_000); + if (remaining <= 0 || Thread.currentThread().isInterrupted()) throw new Failure("IMAGE_DOWNLOAD_TIMEOUT"); + HttpGet request = new HttpGet(uri); + request.setConfig(RequestConfig.custom().setConnectTimeout(Math.min(10000, remaining)) + .setSocketTimeout(Math.min(15000, remaining)).setConnectionRequestTimeout(Math.min(10000, remaining)).build()); + request.setHeader("User-Agent", "vibemake-image-migration/1"); + request.setHeader("Accept", "image/*"); + java.util.concurrent.ScheduledFuture timeout = TIMEOUTS.schedule(request::abort, remaining, java.util.concurrent.TimeUnit.MILLISECONDS); + try (CloseableHttpResponse response = client.execute(request)) { + int status = response.getStatusLine().getStatusCode(); + if (status >= 300 && status < 400) { + if (response.getFirstHeader("Location") == null) throw new Failure("IMAGE_REDIRECT_INVALID"); + URI next = uri.resolve(response.getFirstHeader("Location").getValue()); + if ("https".equalsIgnoreCase(uri.getScheme()) && !"https".equalsIgnoreCase(next.getScheme())) + throw new Failure("IMAGE_REDIRECT_DOWNGRADE"); + uri = next; + request.abort(); + continue; + } + if (status != 200) throw new Failure("IMAGE_HTTP_" + status); + if (response.getEntity() == null) throw new Failure("IMAGE_EMPTY"); + if (response.getEntity().getContentLength() > maxBytes) throw new Failure("IMAGE_TOO_LARGE"); + try (InputStream input = response.getEntity().getContent()) { + return image(read(input, maxBytes, deadline), true); + } finally { request.abort(); } + } finally { timeout.cancel(false); request.abort(); } + } + } + throw new Failure("IMAGE_REDIRECT_LIMIT"); + } + + public static void validateUri(URI uri) { + String scheme = uri.getScheme(); + if (!("https".equalsIgnoreCase(scheme) || "http".equalsIgnoreCase(scheme)) || uri.getHost() == null + || uri.getRawUserInfo() != null || (uri.getPort() != -1 && uri.getPort() != ("https".equalsIgnoreCase(scheme) ? 443 : 80))) + throw new Failure("IMAGE_URL_INVALID"); + } + + public static void validateAddress(InetAddress address) { + byte[] bytes = address.getAddress(); + boolean reservedIpv4 = bytes.length == 4 && ((bytes[0] & 255) == 0 || (bytes[0] & 255) >= 224 + || (bytes[0] & 255) == 100 && ((bytes[1] & 255) & 192) == 64 + || (bytes[0] & 255) == 198 && ((bytes[1] & 255) == 18 || (bytes[1] & 255) == 19)); + // 只允许 IPv6 全球单播 2000::/3,包含 IPv4 映射时 InetAddress 会还原为 IPv4。 + boolean reservedIpv6 = bytes.length == 16 && (bytes[0] & 224) != 32; + if (address.isAnyLocalAddress() || address.isLoopbackAddress() || address.isLinkLocalAddress() + || address.isSiteLocalAddress() || address.isMulticastAddress() || reservedIpv4 || reservedIpv6) + throw new Failure("IMAGE_PRIVATE_ADDRESS"); + } + + public static byte[] read(InputStream input, int limit, long deadline) throws Exception { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int count; + while ((count = input.read(buffer)) != -1) { + if (System.nanoTime() > deadline || Thread.currentThread().isInterrupted()) throw new Failure("IMAGE_DOWNLOAD_TIMEOUT"); + if (count > limit - output.size()) throw new Failure("IMAGE_TOO_LARGE"); + output.write(buffer, 0, count); + } + return output.toByteArray(); + } + + public static Image image(byte[] bytes, boolean allowSvg) throws Exception { + if (bytes.length >= 3 && (bytes[0] & 255) == 255 && (bytes[1] & 255) == 216 && (bytes[2] & 255) == 255) + return new Image(bytes, "image/jpeg", "jpg"); + if (bytes.length >= 8 && Arrays.equals(Arrays.copyOf(bytes, 8), new byte[] {(byte)137,80,78,71,13,10,26,10})) + return new Image(bytes, "image/png", "png"); + if (bytes.length >= 6 && new String(bytes, 0, 6, StandardCharsets.US_ASCII).matches("GIF8[79]a")) + return new Image(bytes, "image/gif", "gif"); + if (bytes.length >= 12 && new String(bytes, 0, 4, StandardCharsets.US_ASCII).equals("RIFF") + && new String(bytes, 8, 4, StandardCharsets.US_ASCII).equals("WEBP")) + return new Image(bytes, "image/webp", "webp"); + if (allowSvg && new String(bytes, StandardCharsets.UTF_8).stripLeading().matches("(?s)(?:\\uFEFF)?<.*")) { + byte[] safe = stripExternalSvgDoctype(bytes); + validateSvg(safe); + return new Image(safe, "image/svg+xml", "svg"); + } + throw new Failure("IMAGE_FORMAT_UNSUPPORTED"); + } + + /** 旧 Illustrator/Creative Commons SVG 常带外部 DTD;移除纯外部声明,仍拒绝内部实体。 */ + private static byte[] stripExternalSvgDoctype(byte[] bytes) { + String text = new String(bytes, StandardCharsets.UTF_8); + String quoted = "(?:\"[^\"<>\\[\\]]*\"|'[^'<>\\[\\]]*')"; + java.util.regex.Matcher declaration = java.util.regex.Pattern.compile( + "(?is)").matcher(text); + java.util.regex.Matcher root = java.util.regex.Pattern.compile("(?i)])").matcher(text); + if (root.find() && declaration.find() && declaration.start() < root.start()) + return (text.substring(0, declaration.start()) + text.substring(declaration.end())).getBytes(StandardCharsets.UTF_8); + return bytes; + } + + private static void validateSvg(byte[] bytes) throws Exception { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + factory.setXIncludeAware(false); + factory.setExpandEntityReferences(false); + javax.xml.parsers.DocumentBuilder builder = factory.newDocumentBuilder(); + builder.setErrorHandler(new DefaultHandler()); + Element root; + try { root = builder.parse(new ByteArrayInputStream(bytes)).getDocumentElement(); } + catch (org.xml.sax.SAXException error) { throw new Failure("IMAGE_SVG_UNSAFE"); } + if (!"svg".equals(root.getLocalName()) || !"http://www.w3.org/2000/svg".equals(root.getNamespaceURI())) + throw new Failure("IMAGE_FORMAT_UNSUPPORTED"); + NodeList elements = root.getElementsByTagName("*"); + for (int index = -1; index < elements.getLength(); index++) { + Element element = index == -1 ? root : (Element) elements.item(index); + String name = element.getLocalName(); + if ("script".equalsIgnoreCase(name) || "foreignObject".equalsIgnoreCase(name) + || "animate".equalsIgnoreCase(name) || "set".equalsIgnoreCase(name)) throw new Failure("IMAGE_SVG_UNSAFE"); + NamedNodeMap attributes = element.getAttributes(); + for (int attribute = 0; attribute < attributes.getLength(); attribute++) { + org.w3c.dom.Node item = attributes.item(attribute); + String key = item.getLocalName() == null ? item.getNodeName() : item.getLocalName(); + String value = item.getNodeValue().strip(); + if (key.toLowerCase(Locale.ROOT).startsWith("on") + || "href".equalsIgnoreCase(key) && !value.startsWith("#") + || unsafeCss(value)) throw new Failure("IMAGE_SVG_UNSAFE"); + } + if ("style".equalsIgnoreCase(name) && unsafeCss(element.getTextContent())) throw new Failure("IMAGE_SVG_UNSAFE"); + } + } + + private static boolean unsafeCss(String value) { + String normalized = value.toLowerCase(Locale.ROOT); + if (normalized.contains("@import") || normalized.contains("\\")) return true; + java.util.regex.Matcher urls = java.util.regex.Pattern.compile("url\\(\\s*(['\"]?)(.*?)\\1\\s*\\)", java.util.regex.Pattern.DOTALL).matcher(normalized); + while (urls.find()) if (!urls.group(2).strip().startsWith("#")) return true; + return false; + } +} diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ImageOssUploader.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ImageOssUploader.java new file mode 100644 index 0000000..a1825f7 --- /dev/null +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ImageOssUploader.java @@ -0,0 +1,77 @@ +package com.linkerbot.vibemake.service.readme; + +import com.aliyun.oss.ClientBuilderConfiguration; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.common.auth.DefaultCredentialProvider; +import com.aliyun.oss.common.comm.SignVersion; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.PutObjectRequest; +import com.linkerbot.vibemake.service.config.OssProperties; +import jakarta.annotation.PreDestroy; +import java.io.ByteArrayInputStream; +import java.util.Map; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Component; + +/** 封面与 README 共用的按摘要核验、禁止覆盖上传;客户端按需初始化。 */ +@Component +public class ImageOssUploader implements AutoCloseable { + private final OssProperties properties; + private OSS client; + public ImageOssUploader(OssProperties properties) { this.properties = properties; } + + private synchronized OSS client() { + if (client != null) return client; + OssProperties checked = new OssProperties(); + BeanUtils.copyProperties(properties, checked); + checked.setEnabled(true); + if (!checked.isConfigurationValid()) throw new ExternalImageDownloader.Failure("OSS_CONFIGURATION_MISSING"); + ClientBuilderConfiguration configuration = new ClientBuilderConfiguration(); + configuration.setSignatureVersion(SignVersion.V4); + configuration.setConnectionTimeout(10000); + configuration.setSocketTimeout(20000); + configuration.setRequestTimeoutEnabled(true); + configuration.setRequestTimeout(30000); + configuration.setMaxErrorRetry(1); + client = OSSClientBuilder.create().endpoint(properties.resolvedEndpoint()).region(properties.getRegion()) + .credentialsProvider(new DefaultCredentialProvider(properties.getAccessKeyId(), properties.getAccessKeySecret())) + .clientConfiguration(configuration).build(); + return client; + } + + public void upload(String key, ExternalImageDownloader.Image image, String sha256) { + ObjectMetadata existing = metadata(key); + if (existing != null) { verify(existing, image, sha256); return; } + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(image.bytes().length); + metadata.setContentType(image.contentType()); + metadata.setCacheControl("public, max-age=31536000, immutable"); + metadata.setUserMetadata(Map.of("sha256", sha256)); + metadata.setHeader("x-oss-forbid-overwrite", "true"); + try { + client().putObject(new PutObjectRequest(properties.getBucket(), key, new ByteArrayInputStream(image.bytes()), metadata)); + } catch (OSSException error) { + if (!"FileAlreadyExists".equals(error.getErrorCode())) throw new ExternalImageDownloader.Failure("OSS_PUT_" + error.getErrorCode()); + } + ObjectMetadata uploaded = metadata(key); + if (uploaded == null) throw new ExternalImageDownloader.Failure("OSS_VERIFY_MISSING"); + verify(uploaded, image, sha256); + } + + private ObjectMetadata metadata(String key) { + try { return client().getObjectMetadata(properties.getBucket(), key); } + catch (OSSException error) { + if ("NoSuchKey".equals(error.getErrorCode())) return null; + throw new ExternalImageDownloader.Failure("OSS_HEAD_" + error.getErrorCode()); + } + } + + private void verify(ObjectMetadata metadata, ExternalImageDownloader.Image image, String digest) { + if (metadata.getContentLength() != image.bytes().length || !digest.equals(metadata.getUserMetadata().get("sha256")) + || !image.contentType().equals(metadata.getContentType())) throw new ExternalImageDownloader.Failure("OSS_VERIFY_MISMATCH"); + } + + @Override @PreDestroy public synchronized void close() { if (client != null) { client.shutdown(); client = null; } } +} diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ReadmeImages.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ReadmeImages.java new file mode 100644 index 0000000..d3295b5 --- /dev/null +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ReadmeImages.java @@ -0,0 +1,225 @@ +package com.linkerbot.vibemake.service.readme; + +import com.linkerbot.vibemake.service.config.OssProperties; +import com.linkerbot.vibemake.service.config.ReadmeImageProperties; +import com.linkerbot.vibemake.service.oss.OssResourcePaths; +import com.vladsch.flexmark.ast.*; +import com.vladsch.flexmark.parser.Parser; +import com.vladsch.flexmark.util.ast.Document; +import com.vladsch.flexmark.util.ast.Node; +import com.vladsch.flexmark.util.sequence.BasedSequence; +import com.vladsch.flexmark.util.sequence.Escaping; +import java.net.URI; +import java.security.MessageDigest; +import java.util.*; +import java.util.concurrent.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** 解析和原位替换,不经过 Markdown/HTML 全文重新渲染。 */ +@Component +public class ReadmeImages { + public record Reference(int start, int end, String url, String prefix, String suffix, boolean html) { + public Reference(int start, int end, String url, String prefix, String suffix) { + this(start, end, url, prefix, suffix, false); + } + } + public record Scan(List images, int external, int local, int managed, int badges) {} + public record Result(String content, int replaced, int failed, int badges, List warnings) {} + @FunctionalInterface public interface Transfer { String store(String url) throws Exception; } + + private static final Logger log = LoggerFactory.getLogger(ReadmeImages.class); + private static final Pattern PROTECTED_HTML = Pattern.compile("(?is)|<(pre|code|script|style)\\b[^>]*>.*?"); + private static final Pattern TAG = Pattern.compile("(?is)<(img|source)\\b(?:[^>\"']|\"[^\"]*\"|'[^']*')*>"); + private static final Pattern ATTRIBUTE = Pattern.compile("([\\w:-]+)(?:\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s>]+)))?"); + private final ReadmeImageProperties settings; + private final OssProperties oss; + private final Transfer transfer; + private final Map successful = new ConcurrentHashMap<>(); + + @org.springframework.beans.factory.annotation.Autowired + public ReadmeImages(ReadmeImageProperties settings, OssProperties oss, OssResourcePaths paths, ImageOssUploader uploader) { + this(settings, oss, url -> { + ExternalImageDownloader downloader = new ExternalImageDownloader(settings.getMaxBytes()); + ExternalImageDownloader.Image image = downloader.download(url); + String digest = digest(image.bytes()); + String path = "/readme-images/" + digest + "." + image.extension(); + uploader.upload(paths.objectKey(path), image, digest); + String publicUrl = paths.publicUrl(path); + if (!digest.equals(digest(downloader.download(publicUrl).bytes()))) + throw new ExternalImageDownloader.Failure("OSS_PUBLIC_VERIFY_MISMATCH"); + return publicUrl; + }); + } + + public ReadmeImages(ReadmeImageProperties settings, OssProperties oss, Transfer transfer) { + this.settings = settings; + this.oss = oss; + this.transfer = transfer; + } + + public Scan scan(String content) { + Document document = Parser.builder().build().parse(content); + List references = new ArrayList<>(); + Set unique = new LinkedHashSet<>(); + Set linked = new HashSet<>(); + for (Node node : document.getDescendants()) + if (node instanceof LinkRef link && link.getReferenceNode(document) != null) linked.add(link.getReferenceNode(document)); + List protectedRanges = new ArrayList<>(); + Matcher protectedHtml = PROTECTED_HTML.matcher(content); + while (protectedHtml.find()) protectedRanges.add(new int[] {protectedHtml.start(), protectedHtml.end()}); + for (Node node : document.getDescendants()) { + if (protectedRanges.stream().anyMatch(range -> node.getStartOffset() >= range[0] && node.getStartOffset() < range[1])) continue; + if (node instanceof Image image) add(unique, image.getUrl()); + else if (node instanceof ImageRef image) { + com.vladsch.flexmark.ast.Reference definition = image.getReferenceNode(document); + if (definition == null) continue; + if (!linked.contains(definition)) add(unique, definition.getUrl()); + else { + // 图片与普通链接共享定义时,只将图片改为内联写法,普通链接仍指向原地址。 + String prefix = content.substring(image.getStartOffset(), image.getTextClosingMarker().getEndOffset()) + "(<"; + String title = definition.getTitleOpeningMarker().isNotNull() + ? " " + definition.getTitleOpeningMarker() + definition.getTitle() + definition.getTitleClosingMarker() : ""; + unique.add(new Reference(image.getStartOffset(), image.getEndOffset(), Escaping.unescapeString(definition.getUrl().toString()), prefix, ">" + title + ")")); + } + } else if (node instanceof HtmlInline || node instanceof HtmlBlock) { + Matcher tags = TAG.matcher(node.getChars()); + while (tags.find()) { + int offset = node.getStartOffset() + tags.start(); + if (protectedRanges.stream().anyMatch(range -> offset >= range[0] && offset < range[1])) continue; + Matcher attributes = ATTRIBUTE.matcher(tags.group()); + attributes.region(tags.group(1).length() + 1, tags.group().length()); + while (attributes.find()) { + String name = attributes.group(1).toLowerCase(Locale.ROOT); + int group = attributes.group(2) != null ? 2 : attributes.group(3) != null ? 3 : 4; + String value = attributes.group(group); + if (value == null) continue; + int start = offset + attributes.start(group); + if ("src".equals(name) && "img".equalsIgnoreCase(tags.group(1))) + unique.add(new Reference(start, start + value.length(), Escaping.unescapeHtml(value), "", "", true)); + else if ("srcset".equals(name)) srcset(unique, value, start); + } + } + } + } + int external = 0, local = 0, managed = 0, badges = 0; + for (Reference reference : unique) { + String url = reference.url(); + if (!url.matches("(?is)^(?:https?:)?//.*")) { local++; continue; } + external++; + if (managed(url)) { managed++; continue; } + if (badge(url)) { badges++; continue; } + references.add(reference); + } + references.sort(Comparator.comparingInt(Reference::start)); + return new Scan(List.copyOf(references), external, local, managed, badges); + } + + private static void add(Set references, BasedSequence value) { + references.add(new Reference(value.getStartOffset(), value.getEndOffset(), Escaping.unescapeString(value.toString()), "", "")); + } + + private static void srcset(Set references, String value, int offset) { + int index = 0; + while (index < value.length()) { + while (index < value.length() && (Character.isWhitespace(value.charAt(index)) || value.charAt(index) == ',')) index++; + int start = index; + while (index < value.length() && !Character.isWhitespace(value.charAt(index))) index++; + int end = index; + while (end > start && value.charAt(end - 1) == ',') end--; + if (end > start) references.add(new Reference(offset + start, offset + end, Escaping.unescapeHtml(value.substring(start, end)), "", "", true)); + if (end != index) continue; + while (index < value.length() && value.charAt(index) != ',') index++; + } + } + + public Result replace(String projectCode, String content) { + Scan scan = scan(content); + if (!settings.isEnabled()) return new Result(content, 0, scan.images().size(), scan.badges(), + scan.images().isEmpty() ? List.of() : List.of("README 图片转存未启用,保留原链")); + Map> jobs = new LinkedHashMap<>(); + try (ExecutorService executor = Executors.newFixedThreadPool(Math.max(1, Math.min(4, settings.getConcurrency())))) { + for (Reference reference : scan.images()) jobs.computeIfAbsent(reference.url(), url -> executor.submit(() -> { + String cached = successful.get(url); + if (cached != null) return cached; + String stored = transfer.store(url); + log.info("README 图片已转存 projectCode={} sourceHost={}", projectCode, host(url)); + if (successful.size() < 4096) successful.put(url, stored); + return stored; + })); + Map replacements = new HashMap<>(); + List warnings = new ArrayList<>(); + for (Map.Entry> job : jobs.entrySet()) { + try { replacements.put(job.getKey(), job.getValue().get()); } + catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("README 图片转存被中断"); + } catch (ExecutionException error) { + Throwable cause = error.getCause(); + String code = cause instanceof ExternalImageDownloader.Failure failure ? failure.code() : cause.getClass().getSimpleName(); + warnings.add("README 图片转存失败 host=" + host(job.getKey()) + " code=" + code); + log.warn("README 图片转存失败 projectCode={} sourceHost={} code={}", projectCode, host(job.getKey()), code); + } + } + StringBuilder result = new StringBuilder(content); + int replaced = 0; + List reverse = new ArrayList<>(scan.images()); + Collections.reverse(reverse); + for (Reference reference : reverse) { + String stored = replacements.get(reference.url()); + if (stored == null) continue; + String fragment = uri(reference.url()).getRawFragment(); + if (fragment != null) stored += "#" + fragment; + if (reference.html()) stored = stored.replace("&", "&").replace("\"", """) + .replace("'", "'").replace("<", "<").replace(">", ">"); + result.replace(reference.start(), reference.end(), reference.prefix() + stored + reference.suffix()); + replaced++; + } + return new Result(result.toString(), replaced, scan.images().size() - replaced, scan.badges(), List.copyOf(warnings)); + } + } + + private boolean managed(String value) { + try { + URI uri = uri(value); + List bases = new ArrayList<>(oss.getLegacyPublicBaseUrls()); + bases.add(oss.publicBaseUrl()); + if (!oss.getBucket().isBlank() && !oss.getRegion().isBlank()) bases.add(oss.uploadUrl()); + for (String baseValue : bases) { + URI base = URI.create(baseValue); + String path = base.getPath().replaceAll("/+$", ""); + if (base.getHost() != null && base.getHost().equalsIgnoreCase(uri.getHost()) + && (uri.getPort() == -1 ? 443 : uri.getPort()) == (base.getPort() == -1 ? 443 : base.getPort()) + && (path.isEmpty() || uri.getPath().equals(path) || uri.getPath().startsWith(path + "/"))) return true; + } + } catch (IllegalArgumentException | NullPointerException ignored) { /* 非法历史 URL 进入失败报告。 */ } + return false; + } + + private boolean badge(String value) { + try { + URI uri = uri(value); + String host = uri.getHost().toLowerCase(Locale.ROOT); + String path = uri.getPath().toLowerCase(Locale.ROOT); + if (host.equals("shields.io") || host.endsWith(".shields.io") + || host.equals("build.ros2.org") || host.equals("build.ros.org") + || host.equals("github.com") && (path.contains("/actions/") && path.endsWith("/badge.svg") || path.endsWith("/workflows/ci/badge.svg")) + || (host.endsWith("travis-ci.org") || host.endsWith("travis-ci.com") || host.equals("codecov.io") || host.equals("coveralls.io")) + || (path.contains("/badge/") || path.contains("/badges/")) && path.endsWith(".svg")) return true; + return settings.getPreserveUrlPatterns().stream().anyMatch(pattern -> Pattern.matches(pattern, value)); + } catch (IllegalArgumentException | NullPointerException ignored) { return false; } + } + + private static URI uri(String value) { return URI.create(value.startsWith("//") ? "https:" + value : value); } + public static String host(String value) { + try { String host = uri(value).getHost(); return host == null ? "invalid" : host; } + catch (IllegalArgumentException error) { return "invalid"; } + } + public static String digest(byte[] bytes) { + try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); } + catch (java.security.NoSuchAlgorithmException error) { throw new IllegalStateException(error); } + } +} diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ReadmeMigrationReceipt.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ReadmeMigrationReceipt.java new file mode 100644 index 0000000..e40d24e --- /dev/null +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ReadmeMigrationReceipt.java @@ -0,0 +1,27 @@ +package com.linkerbot.vibemake.service.readme; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkerbot.vibemake.service.gitea.GiteaClient; +import java.io.IOException; + +/** 兼容旧收据,在既有 JSON 中保存 README 进度,来源 SHA 不被图片提交覆盖。 */ +public final class ReadmeMigrationReceipt { + private ReadmeMigrationReceipt() {} + public static GiteaClient.RemoteRepository repository(ObjectMapper json, String value) throws IOException { + ObjectNode node = (ObjectNode) json.readTree(value); + node.remove("readmeImages"); + return json.treeToValue(node, GiteaClient.RemoteRepository.class); + } + public static ReadmeRepository.State state(ObjectMapper json, String value) throws IOException { + if (value == null) return null; + JsonNode node = json.readTree(value).get("readmeImages"); + return node == null || node.isNull() ? null : json.treeToValue(node, ReadmeRepository.State.class); + } + public static String encode(ObjectMapper json, GiteaClient.RemoteRepository repository, ReadmeRepository.State state) throws IOException { + ObjectNode node = json.valueToTree(repository); + node.set("readmeImages", json.valueToTree(state)); + return json.writeValueAsString(node); + } +} diff --git a/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ReadmeRepository.java b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ReadmeRepository.java new file mode 100644 index 0000000..5f41d34 --- /dev/null +++ b/vibemake-service/src/main/java/com/linkerbot/vibemake/service/readme/ReadmeRepository.java @@ -0,0 +1,154 @@ +package com.linkerbot.vibemake.service.readme; + +import com.linkerbot.vibemake.bean.Contracts.TreeEntry; +import com.linkerbot.vibemake.service.gitea.GiteaClient; +import com.linkerbot.vibemake.service.gitea.GiteaFailure; +import com.linkerbot.vibemake.service.gitea.WriteReceipt; +import java.nio.ByteBuffer; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** 调用方负责项目锁和状态持久化;状态先落盘,再进行任何 Git 写入。 */ +@Component +public class ReadmeRepository { + public static class State { + public String branch; + public String baseSha; + public String path; + public String blobSha; + public String operationCode; + public String digest; + public String resultSha; + public String stage; + public int replaced; + public int failed; + public int badges; + public List warnings = new ArrayList<>(); + } + public record Prepared(String path, String blobSha, String original, ReadmeImages.Scan scan, ReadmeImages.Result result) {} + @FunctionalInterface public interface Checkpoint { void save(State state); } + private static final Logger log = LoggerFactory.getLogger(ReadmeRepository.class); + private final GiteaClient gitea; + private final ReadmeImages images; + public ReadmeRepository(GiteaClient gitea, ReadmeImages images) { this.gitea = gitea; this.images = images; } + + public Prepared prepare(String code, GiteaClient.Repository repository, String sha, boolean execute) { + try { + List entries = gitea.tree(repository, sha, ""); + List readmes = entries.stream().filter(entry -> "file".equals(entry.type()) && "readme.md".equalsIgnoreCase(entry.name())).toList(); + if (readmes.isEmpty()) return new Prepared(null, null, "", images.scan(""), new ReadmeImages.Result("", 0, 0, 0, List.of())); + // 同时出现两个仅大小写不同的 README 时不猜测前端选择。 + if (readmes.size() != 1) throw new ExternalImageDownloader.Failure("README_AMBIGUOUS"); + GiteaClient.StoredFile file = gitea.file(repository, sha, readmes.getFirst().path()); + String content = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT).decode(ByteBuffer.wrap(file.bytes())).toString(); + ReadmeImages.Scan scan = images.scan(content); + ReadmeImages.Result result = execute ? images.replace(code, content) : new ReadmeImages.Result(content, 0, 0, scan.badges(), List.of()); + return new Prepared(file.path(), file.blobSha(), content, scan, result); + } catch (java.nio.charset.CharacterCodingException | RuntimeException error) { + if (Thread.currentThread().isInterrupted()) throw new IllegalStateException("README 检查被中断"); + String failure = error instanceof com.linkerbot.vibemake.common.BusinessException business ? business.code() + : error instanceof ExternalImageDownloader.Failure image ? image.code() : error.getClass().getSimpleName(); + log.warn("README 检查未完成 projectCode={} commitSha={} code={}", code, sha, failure); + return new Prepared(null, null, "", images.scan(""), new ReadmeImages.Result("", 0, 1, 0, + List.of("README 检查未完成 code=" + failure + ",保留原文"))); + } + } + + public State apply(String code, GiteaClient.Repository repository, String base, String branch, + Prepared prepared, State previous, Checkpoint checkpoint) { + if (previous != null) return resume(code, repository, previous, checkpoint); + State state = new State(); + state.branch = branch; + state.baseSha = base; + state.path = prepared.path(); + state.blobSha = prepared.blobSha(); + state.replaced = prepared.result().replaced(); + state.failed = prepared.result().failed(); + state.badges = prepared.result().badges(); + state.warnings = new ArrayList<>(prepared.result().warnings()); + if (prepared.original().equals(prepared.result().content())) { + state.resultSha = base; + state.stage = "DONE"; + checkpoint.save(state); + return state; + } + state.operationCode = UUID.randomUUID().toString().replace("-", ""); + state.digest = ReadmeImages.digest(prepared.result().content().getBytes(StandardCharsets.UTF_8)); + state.stage = "BRANCH"; + checkpoint.save(state); + ensureBranch(repository, state); + if (!base.equals(gitea.branch(repository, branch))) throw conflict("README_BASE_CHANGED"); + state.stage = "COMMITTING"; + checkpoint.save(state); + try { + state.resultSha = gitea.commit(repository, branch, + List.of(new GiteaClient.Change("update", state.path, state.blobSha, prepared.result().content().getBytes(StandardCharsets.UTF_8))), + "转存 README 外链图片至 OSS" + WriteReceipt.suffix(state.operationCode, state.digest), "Vibemake", "readme-images").commitSha(); + } catch (GiteaFailure error) { + if (!error.uncertain()) { + state.stage = "REJECTED"; + checkpoint.save(state); + throw error; + } + return resume(code, repository, state, checkpoint); + } + verify(repository, state); + state.stage = "DONE"; + checkpoint.save(state); + log.info("README 图片提交完成 projectCode={} branch={} baseSha={} commitSha={} replaced={} failed={}", + code, branch, base, state.resultSha, state.replaced, state.failed); + return state; + } + + public State resume(String code, GiteaClient.Repository repository, State state, Checkpoint checkpoint) { + if ("REJECTED".equals(state.stage)) throw conflict("README_COMMIT_REJECTED"); + if ("BRANCH".equals(state.stage)) { + // 即使没有提交请求记录,也不能在恢复时丢弃已保存的操作意图。 + throw unknown(); + } + if (!Objects.equals(state.baseSha, state.resultSha)) { + String receipt = gitea.findWriteReceipt(repository, state.branch, state.baseSha, state.operationCode, state.digest); + if (receipt == null) throw unknown(); + if (state.resultSha != null && !state.resultSha.equals(receipt)) throw conflict("README_RECEIPT_MISMATCH"); + state.resultSha = receipt; + verify(repository, state); + } + state.stage = "DONE"; + checkpoint.save(state); + log.info("README 图片提交已核对 projectCode={} commitSha={}", code, state.resultSha); + return state; + } + + private void ensureBranch(GiteaClient.Repository repository, State state) { + if ("main".equals(state.branch)) return; + try { + if (!state.baseSha.equals(gitea.branch(repository, state.branch))) throw conflict("README_BRANCH_CONFLICT"); + } catch (GiteaFailure missing) { + if (missing.status() != 404) throw missing; + gitea.createBranch(repository, state.branch, state.baseSha); + } + } + + public void verify(GiteaClient.Repository repository, State state) { + if (state.resultSha == null) throw unknown(); + if (state.baseSha.equals(state.resultSha)) return; + if (!state.digest.equals(ReadmeImages.digest(gitea.file(repository, state.resultSha, state.path).bytes()))) + throw conflict("README_CONTENT_MISMATCH"); + Map before = treeWithoutReadme(repository, state.baseSha, state.path); + Map after = treeWithoutReadme(repository, state.resultSha, state.path); + if (!before.equals(after)) throw conflict("README_UNEXPECTED_FILES"); + } + + private Map treeWithoutReadme(GiteaClient.Repository repository, String sha, String path) { + Map result = new TreeMap<>(); + for (TreeEntry entry : gitea.tree(repository, sha, "")) if (!entry.path().equals(path)) result.put(entry.path(), entry); + return result; + } + private static GiteaFailure conflict(String code) { return new GiteaFailure(409, code, "README 处理版本或收据发生变化", false); } + private static GiteaFailure unknown() { return new GiteaFailure(503, "README_RESULT_UNKNOWN", "README 提交结果待核对,禁止重发", true); } +} diff --git a/vibemake-service/src/test/java/com/linkerbot/vibemake/service/readme/ExternalImageDownloaderTest.java b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/readme/ExternalImageDownloaderTest.java new file mode 100644 index 0000000..5106c56 --- /dev/null +++ b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/readme/ExternalImageDownloaderTest.java @@ -0,0 +1,40 @@ +package com.linkerbot.vibemake.service.readme; + +import static org.assertj.core.api.Assertions.*; +import java.net.InetAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; + +class ExternalImageDownloaderTest { + @Test void validatesSvgWithoutExternalEntitiesScriptsOrRemoteResources() throws Exception { + String svg = ""; + assertThat(ExternalImageDownloader.image(svg.getBytes(StandardCharsets.UTF_8), true).extension()).isEqualTo("svg"); + for (String unsafe : new String[] { + "]>&x;", + "", + "", + "", + ""}) { + assertThatThrownBy(() -> ExternalImageDownloader.image(unsafe.getBytes(StandardCharsets.UTF_8), true)) + .isInstanceOf(ExternalImageDownloader.Failure.class).hasMessage("IMAGE_SVG_UNSAFE"); + } + } + + @Test void rejectsPrivateAndReservedAddressesAndCredentialBearingUrls() throws Exception { + for (String address : new String[] {"127.0.0.1", "10.1.2.3", "169.254.169.254", "100.64.0.1", "198.18.0.1", "::1", "fc00::1", "::ffff:127.0.0.1"}) + assertThatThrownBy(() -> ExternalImageDownloader.validateAddress(InetAddress.getByName(address))) + .hasMessage("IMAGE_PRIVATE_ADDRESS"); + ExternalImageDownloader.validateAddress(InetAddress.getByName("8.8.8.8")); + assertThatThrownBy(() -> ExternalImageDownloader.validateUri(URI.create("https://user:secret@example.com/a.png"))) + .hasMessage("IMAGE_URL_INVALID"); + } + + @Test void legacySvgExternalDoctypeIsRemovedWithoutResolvingIt() throws Exception { + String svg = "" + + ""; + ExternalImageDownloader.Image image = ExternalImageDownloader.image(svg.getBytes(StandardCharsets.UTF_8), true); + assertThat(image.extension()).isEqualTo("svg"); + assertThat(new String(image.bytes(), StandardCharsets.UTF_8)).doesNotContain("DOCTYPE", "127.0.0.1").contains(""); + } +} diff --git a/vibemake-service/src/test/java/com/linkerbot/vibemake/service/readme/ReadmeImagesTest.java b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/readme/ReadmeImagesTest.java new file mode 100644 index 0000000..5902c1d --- /dev/null +++ b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/readme/ReadmeImagesTest.java @@ -0,0 +1,80 @@ +package com.linkerbot.vibemake.service.readme; + +import static org.assertj.core.api.Assertions.*; +import com.linkerbot.vibemake.service.config.OssProperties; +import com.linkerbot.vibemake.service.config.ReadmeImageProperties; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +class ReadmeImagesTest { + private OssProperties oss() { + OssProperties properties = new OssProperties(); + properties.setPublicBaseUrl("https://assets.example.com"); + properties.setRootDirectory("vibemake"); + return properties; + } + + @Test void replacesOnlyImageDestinationsAndKeepsFormattingCodeLocalLinksAndBadges() { + String original = "\uFEFF# 介绍\r\n![a](https://github.com/user-attachments/assets/a \"图\")\r\n" + + "\"image\"\r\n" + + "![local](./img/a.png) [普通](https://github.com/user-attachments/assets/a)\r\n" + + "![badge](https://img.shields.io/badge/build-pass-green.svg)\r\n" + + "`![code](https://example.com/code.png)`\r\n```html\r\n\r\n```\r\n" + + "\r\n
\r\n"; + AtomicInteger downloads = new AtomicInteger(); + ReadmeImages images = new ReadmeImages(new ReadmeImageProperties(), oss(), url -> { downloads.incrementAndGet(); return "https://assets.example.com/vibemake/readme-images/a.png"; }); + ReadmeImages.Result result = images.replace("project", original); + assertThat(result.replaced()).isEqualTo(2); + assertThat(downloads).hasValue(1); + assertThat(result.content()).isEqualTo(original.replace("![a](https://github.com/user-attachments/assets/a", "![a](https://assets.example.com/vibemake/readme-images/a.png") + .replace("src=\"https://github.com/user-attachments/assets/a\"", "src=\"https://assets.example.com/vibemake/readme-images/a.png\"")); + assertThat(images.scan(result.content()).images()).isEmpty(); + assertThat(result.badges()).isEqualTo(1); + } + + @Test void handlesReferenceImagesWithoutChangingSharedOrdinaryLinks() { + ReadmeImages images = new ReadmeImages(new ReadmeImageProperties(), oss(), url -> "https://assets.example.com/a.png"); + String original = "![one][image]\n\n[image]: https://example.com/a.png \"原图\"\n\n![two][shared] [普通][shared]\n\n[shared]: https://example.com/b.png \"标题\"\n"; + ReadmeImages.Result result = images.replace("project", original); + assertThat(result.content()).contains("[image]: https://assets.example.com/a.png \"原图\"") + .contains("![two]( \"标题\") [普通][shared]") + .contains("[shared]: https://example.com/b.png \"标题\""); + assertThat(result.replaced()).isEqualTo(2); + } + + @Test void handlesPictureSrcsetHtmlEntitiesAndFailuresWithoutBreakingOtherImages() { + ReadmeImages images = new ReadmeImages(new ReadmeImageProperties(), oss(), url -> { + if (url.contains("fail")) throw new ExternalImageDownloader.Failure("IMAGE_HTTP_404"); + return "https://assets.example.com/" + (url.contains("large") ? "large.png" : "small.png"); + }); + String original = "" + + ""; + ReadmeImages.Result result = images.replace("project", original); + assertThat(result.replaced()).isEqualTo(3); + assertThat(result.failed()).isEqualTo(1); + assertThat(result.content()).contains("src='https://example.com/fail.png'") + .contains("srcset=\"https://assets.example.com/small.png 1x, https://assets.example.com/large.png 2x\"") + .contains("title='src=\"https://example.com/no.png\"'") + .contains("local.png 1x"); + assertThat(result.warnings()).allMatch(warning -> !warning.contains("?")); + } + + @Test void supportsCustomBadgeRulesAndSkipsManagedCdn() { + ReadmeImageProperties settings = new ReadmeImageProperties(); + settings.setPreserveUrlPatterns(java.util.List.of("https://status\\.example\\.com/.*")); + ReadmeImages images = new ReadmeImages(settings, oss(), url -> { throw new AssertionError(); }); + ReadmeImages.Scan scan = images.scan("![a](https://assets.example.com/vibemake/a.png) ![b](https://status.example.com/ci.svg) ![c](../a.png)"); + assertThat(scan.images()).isEmpty(); + assertThat(scan.managed()).isEqualTo(1); + assertThat(scan.badges()).isEqualTo(1); + assertThat(scan.local()).isEqualTo(1); + } + + @Test void preservesImageFragmentsAndDoesNotDownloadTwiceOnRepeatedReferences() { + ReadmeImages images = new ReadmeImages(new ReadmeImageProperties(), oss(), url -> "https://assets.example.com/a.svg"); + ReadmeImages.Result result = images.replace("project", "![a](https://example.com/a.svg#dark-mode-only)"); + assertThat(result.content()).isEqualTo("![a](https://assets.example.com/a.svg#dark-mode-only)"); + result = images.replace("project", ""); + assertThat(result.content()).isEqualTo(""); + } +} diff --git a/vibemake-service/src/test/java/com/linkerbot/vibemake/service/readme/ReadmeRepositoryTest.java b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/readme/ReadmeRepositoryTest.java new file mode 100644 index 0000000..3711525 --- /dev/null +++ b/vibemake-service/src/test/java/com/linkerbot/vibemake/service/readme/ReadmeRepositoryTest.java @@ -0,0 +1,82 @@ +package com.linkerbot.vibemake.service.readme; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; +import com.linkerbot.vibemake.bean.Contracts.CommitView; +import com.linkerbot.vibemake.bean.Contracts.TreeEntry; +import com.linkerbot.vibemake.service.config.OssProperties; +import com.linkerbot.vibemake.service.config.ReadmeImageProperties; +import com.linkerbot.vibemake.service.gitea.GiteaClient; +import com.linkerbot.vibemake.service.gitea.GiteaFailure; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +class ReadmeRepositoryTest { + private static final String BASE = "a".repeat(40); + private static final String NEXT = "b".repeat(40); + private final GiteaClient remote = mock(GiteaClient.class); + private final GiteaClient.Repository repository = new GiteaClient.Repository(1, "org", "repo"); + private final AtomicReference changed = new AtomicReference<>(); + + private ReadmeRepository worker() { + OssProperties oss = new OssProperties(); + oss.setPublicBaseUrl("https://assets.example.com"); + ReadmeImages images = new ReadmeImages(new ReadmeImageProperties(), oss, url -> "https://assets.example.com/a.png"); + when(remote.tree(eq(repository), anyString(), eq(""))).thenAnswer(call -> List.of( + new TreeEntry("README.md", "README.md", "file", 10, call.getArgument(1).equals(BASE) ? "old" : "new"), + new TreeEntry("code", "code", "dir", 0, "untouched"))); + when(remote.file(eq(repository), eq(BASE), eq("README.md"))).thenReturn(new GiteaClient.StoredFile("README.md", "old", "![a](https://example.com/a.png)".getBytes(StandardCharsets.UTF_8))); + when(remote.file(eq(repository), eq(NEXT), eq("README.md"))).thenAnswer(call -> new GiteaClient.StoredFile("README.md", "new", changed.get())); + when(remote.branch(eq(repository), anyString())).thenReturn(BASE); + return new ReadmeRepository(remote, images); + } + + @Test void durableIntentPrecedesCommitAndResponseLossUsesReceiptWithoutRepeatingWrite() { + ReadmeRepository worker = worker(); + AtomicReference saved = new AtomicReference<>(); + when(remote.commit(eq(repository), eq("main"), anyList(), anyString(), anyString(), anyString())).thenAnswer(call -> { + assertThat(saved.get().stage).isEqualTo("COMMITTING"); + List changes = call.getArgument(2); + changed.set(changes.getFirst().bytes()); + throw new GiteaFailure(504, "REMOTE_RESULT_UNKNOWN", "timeout", true); + }); + when(remote.findWriteReceipt(eq(repository), eq("main"), eq(BASE), anyString(), anyString())).thenReturn(NEXT); + ReadmeRepository.State state = worker.apply("project", repository, BASE, "main", worker.prepare("project", repository, BASE, true), null, saved::set); + assertThat(state.resultSha).isEqualTo(NEXT); + assertThat(state.stage).isEqualTo("DONE"); + worker.apply("project", repository, BASE, "main", null, state, saved::set); + verify(remote, times(1)).commit(eq(repository), eq("main"), anyList(), anyString(), anyString(), anyString()); + } + + @Test void unknownCommitResultIsNeverAutomaticallyReissued() { + ReadmeRepository worker = worker(); + AtomicReference saved = new AtomicReference<>(); + when(remote.commit(eq(repository), eq("main"), anyList(), anyString(), anyString(), anyString())) + .thenThrow(new GiteaFailure(504, "REMOTE_RESULT_UNKNOWN", "timeout", true)); + assertThatThrownBy(() -> worker.apply("project", repository, BASE, "main", worker.prepare("project", repository, BASE, true), null, saved::set)) + .isInstanceOfSatisfying(GiteaFailure.class, error -> assertThat(error.uncertain()).isTrue()); + assertThatThrownBy(() -> worker.apply("project", repository, BASE, "main", null, saved.get(), saved::set)).hasMessageContaining("待核对"); + verify(remote, times(1)).commit(eq(repository), eq("main"), anyList(), anyString(), anyString(), anyString()); + } + + @Test void oldPublishedSnapshotUsesSeparateBranchAndRejectsOtherFileChanges() { + ReadmeRepository worker = worker(); + String branch = "maintenance/readme-oss/project/run"; + when(remote.branch(repository, branch)).thenThrow(new GiteaFailure(404, "NOT_FOUND", "missing", false)).thenReturn(BASE); + when(remote.commit(eq(repository), eq(branch), anyList(), anyString(), anyString(), anyString())).thenAnswer(call -> { + List changes = call.getArgument(2); + changed.set(changes.getFirst().bytes()); + return new CommitView(NEXT, "image repair"); + }); + ReadmeRepository.State state = worker.apply("project", repository, BASE, branch, worker.prepare("project", repository, BASE, true), null, ignored -> {}); + assertThat(state.resultSha).isEqualTo(NEXT); + verify(remote).createBranch(repository, branch, BASE); + verify(remote, never()).commit(eq(repository), eq("main"), anyList(), anyString(), anyString(), anyString()); + when(remote.tree(repository, NEXT, "")).thenReturn(List.of(new TreeEntry("README.md", "README.md", "file", 10, "new"), new TreeEntry("code", "code", "dir", 0, "changed"))); + assertThatThrownBy(() -> worker.verify(repository, state)).isInstanceOfSatisfying(GiteaFailure.class, + error -> assertThat(error.code()).isEqualTo("README_UNEXPECTED_FILES")); + } +}