From f8f2e95dbe598bdf0e703e82b6e5a719019ff110 Mon Sep 17 00:00:00 2001 From: lxp <2770281812@qq.com> Date: Mon, 19 Oct 2026 14:53:06 +0800 Subject: [PATCH] =?UTF-8?q?cover=5Furl=E3=80=81urdf=5Fmain=5Fobject=5Fkey?= =?UTF-8?q?=E3=80=81urdf=5Fprefix=E5=8F=AA=E5=AD=98=E7=9B=B8=E5=AF=B9?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=EF=BC=8C=E4=B8=8D=E5=8C=85=E5=90=AB=E5=9F=9F?= =?UTF-8?q?=E5=90=8D+=E7=8E=AF=E5=A2=83=E6=A0=B9=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 2 + README.md | 31 ++- .../src/main/resources/application.yml | 1 + .../linkerbot/community/api/AiSearchTest.java | 24 +++ .../api/ResourcePathMigrationTest.java | 150 ++++++++++++++ .../linkerbot/community/api/WorkflowTest.java | 52 ++++- .../importer/DatabaseConfiguration.java | 8 +- .../community/importer/ImportApplication.java | 8 + .../community/importer/ImportCoordinator.java | 6 +- .../cover/CoverMigrationApplication.java | 172 +++++++++++++++ .../importer/cover/CoverMigrator.java | 137 ++++++++++++ .../importer/cover/CoverOssUploader.java | 72 +++++++ .../cover/ExternalCoverDownloader.java | 132 ++++++++++++ .../src/main/resources/application.yml | 18 ++ .../importer/ImportCoordinatorTest.java | 27 ++- .../importer/cover/CoverMigratorTest.java | 134 ++++++++++++ .../cover/ExternalCoverDownloaderTest.java | 61 ++++++ .../service/ai/AiProjectAssistant.java | 2 +- .../service/config/OssProperties.java | 9 + .../service/impl/ProjectServiceImpl.java | 31 ++- .../community/service/oss/OssPostSigner.java | 4 +- .../service/oss/OssResourcePaths.java | 121 +++++++++++ .../community/service/oss/UrdfResources.java | 41 ++-- .../service/support/ProjectInputs.java | 6 +- .../service/ProjectCodeAllocationTest.java | 3 + .../service/fork/ProjectForkStateTest.java | 4 +- .../service/oss/OssPropertiesTest.java | 17 ++ .../service/oss/OssResourcePathsTest.java | 89 ++++++++ .../service/oss/UrdfResourcesTest.java | 54 ++++- docs/community-api.md | 7 +- docs/database.md | 195 +++++++++++++++++- scripts/migrate-project-covers.sh | 14 ++ scripts/resource-path-rollback.py | 67 ++++++ scripts/tests/test_resource_path_rollback.py | 59 ++++++ 34 files changed, 1700 insertions(+), 58 deletions(-) create mode 100644 community-api/src/test/java/com/linkerbot/community/api/ResourcePathMigrationTest.java create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/cover/CoverMigrationApplication.java create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/cover/CoverMigrator.java create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/cover/CoverOssUploader.java create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/cover/ExternalCoverDownloader.java create mode 100644 community-import/src/test/java/com/linkerbot/community/importer/cover/CoverMigratorTest.java create mode 100644 community-import/src/test/java/com/linkerbot/community/importer/cover/ExternalCoverDownloaderTest.java create mode 100644 community-service/src/main/java/com/linkerbot/community/service/oss/OssResourcePaths.java create mode 100644 community-service/src/test/java/com/linkerbot/community/service/oss/OssResourcePathsTest.java create mode 100644 scripts/migrate-project-covers.sh create mode 100644 scripts/resource-path-rollback.py create mode 100644 scripts/tests/test_resource_path_rollback.py diff --git a/.env.example b/.env.example index 3c61113..c0b8a9a 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,8 @@ OSS_ACCESS_KEY_ID='' OSS_ACCESS_KEY_SECRET='' # 可选,固定公开 CDN/OSS 基础地址;留空使用 Bucket 的标准 HTTPS 域名。 OSS_PUBLIC_BASE_URL='' +# 可选,历史公开基础地址,多个使用英文逗号分隔;不含环境根目录或查询参数。 +OSS_LEGACY_PUBLIC_BASE_URLS='' # URDF 直传限制:单文件 50 MiB、整包 200 MiB、最多 500 个文件,不受仓库 multipart 限制影响。 URDF_MAX_FILE_BYTES=52428800 URDF_MAX_TOTAL_BYTES=209715200 diff --git a/README.md b/README.md index deeb84f..ab863d5 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ python3 scripts/test-gitea.py ## 项目封面与 OSS 直传 -项目通过 `coverUrl` 保存固定公开图片地址,支持 OSS/CDN 或外部 HTTPS 图片。无封面时返回 null;创建时可省略,修改时省略或 null 保留原值,传空字符串清除。Fork 继承同一地址,替换、清除或删除项目不自动删除 OSS 图片。 +项目 API 的 `coverUrl` 接收和返回完整 HTTPS 地址,平台封面在数据库中保存 `/project-covers/...` 环境相对路径,外部图片保留完整 URL。无封面时返回 null;创建时可省略,修改时省略或 null 保留原值,传空字符串清除。Fork 继承同一资源引用,替换、清除或删除项目不自动删除 OSS 图片。URDF 同样保存 `/urdf/...` 环境相对路径,上传和 OSS 操作仍使用含环境根目录的真实 key。人工迁移和回退见 [数据库说明](docs/database.md#资源路径历史数据迁移)。 已有数据库必须先执行 [封面字段升级 SQL](docs/database.md#已有数据库增加项目封面),再部署新应用;新建数据库使用更新后的建表 SQL。应用不会自动变更表结构。 @@ -159,13 +159,38 @@ Fork 不受普通在线文件读取的10 MiB限制,文件浏览、下载和 LF ## 项目短编码与 GitHub 初始化导入 +### 历史外链封面转存 OSS + +`scripts/migrate-project-covers.sh` 读取本项目 `.env`,将数据库现有外链图片下载并转存至当前环境 OSS,上传并核验成功后才更新 `cover_url`。配置 `DB_URL`、`DB_USERNAME`、`DB_PASSWORD`、`OSS_REGION`、`OSS_BUCKET`、`OSS_ACCESS_KEY_ID`、`OSS_ACCESS_KEY_SECRET`;带 `&` 的 JDBC 地址必须加引号。环境根目录沿用 test 的 `vibemake_dev` 或 prod 的 `vibemake`。 + +```bash +# 默认只读预览,不下载、不上传、不更新数据库。 +bash scripts/migrate-project-covers.sh --dry-run +# 先迁移三个项目,核对新封面的访问效果。 +bash scripts/migrate-project-covers.sh --execute --limit 3 +# 迁移其余外链,默认并发 2。 +bash scripts/migrate-project-covers.sh --execute +``` + +支持 JPG、PNG、GIF、WebP,按实际文件头识别类型,原图不压缩、不转码。默认单图最多 5 MiB,可用 `--max-bytes` 调整,上限 50 MiB;`--concurrency` 范围为 1~4。下载只允许公开 HTTPS 地址,每次跳转重新校验,不携带任何平台或 GitHub 凭据,不访问内网地址。失效、超时、超限及非图片资源保留原值并进入失败报告。 + +实际对象位于 `{环境根目录}/project-covers/migrated/{图片SHA256}.{扩展名}`;数据库只保存 `/project-covers/migrated/...`。相同外链只下载一次,相同图片内容复用对象,已转存记录在再次运行时自动跳过;已属于本平台的完整 URL 也跳过,可按数据库迁移说明归一化。扫描包含已逻辑删除项目,以完成历史数据统一。未来新提交外链的行为不由此批处理修改。 + +每次运行在 `logs/cover-migration/` 生成独立报告目录(可用 `--report-dir` 修改)。`preview.json`、`report.json` 只包含项目编码、来源主机、状态及错误码,不记录完整外链或查询参数。`backup.jsonl` 保存修改前原值和拟写入路径,更新前强制落盘;目录和备份文件分别仅允许当前用户访问。原值可能包含签名参数,应作为受限数据库备份保存,不上传仓库或发送聊天。回写通过项目编码和原始 URL 字节比较,跳过迁移期间被修改的封面,不改 `updated_at`,不覆盖其他字段。 + +失败时可以重跑;上传成功但回写未完成的对象也按内容哈希复用,不会自动删除可能共享的对象。退出码 0 表示成功或仅预览,1 表示配置、连接等整体错误,2 表示存在逐项失败。人工恢复时仅将仍等于 `newCoverPath` 的记录按项目编码还原为 `originalCoverUrl`,避免覆盖之后的编辑。 + +对已经核验过的失效链接,可使用 `--source-overrides /绝对路径/备用源.json` 指定备用下载地址,JSON 格式为 `{"项目编码":"https://可信图片地址"}`。下载走同一套 HTTPS、大小和图片类型校验;数据库条件比较及原值备份仍使用原来的外链。采用的备用源另存于受限权限的 `source-overrides-backup.json`,便于审计。已成功迁移的项目在重跑时照常跳过。网络中断最多重试三次,证书校验失败不会绕过验证。 + +### 项目编码与导入流程 + 普通创建、Fork 和导入的新项目使用 12 位小写项目编码,字符集为 `0123456789abcdefghjkmnpqrstvwxyz`。普通项目用安全随机数生成,项目编码唯一约束冲突最多重试 5 次;其他约束错误不重试。导入编码由 `github-import:v1:` 的 SHA-256 前 60 位确定,避免重跑或仓库更名后重复创建。Gitea 仓库名称为 `p_{projectCode}`。历史 32 位编码、仓库及链接继续使用,数据库无需改表。 导入工具仅接受以下固定 Sheet:`1_轮式机器人`、`2_四足机器人`、`3_小人形机器人`、`4_大人形机器人`、`5_机械臂`、`6_其他类型`、`7_开源硬件`。只选 I 列精确为“可访问”、L 列有合法 HTTPS 封面、B 列为具体 GitHub 仓库的行。其他 Sheet 不参与导入;K 列图片公式不会执行。原始 A~H 数据列与表头存在错位,工具按实际位置读取,忽略 H 列的无意义备注 `100`,不将“待查”等信息改写成已确认。 -封面保留原 URL 和 GIF,不转存 OSS。描述由原中文简介和技术资料组合,不调用 AI。轮式、四足、人形分别映射对应类型,机械臂映射 ROBOTIC_ARM,其他和硬件映射 GENERAL,原始分类保留在描述里。 +外部封面保留原 URL 和 GIF,不转存 OSS;当前环境的平台封面统一转换为相对路径。描述由原中文简介和技术资料组合,不调用 AI。轮式、四足、人形分别映射对应类型,机械臂映射 ROBOTIC_ARM,其他和硬件映射 GENERAL,原始分类保留在描述里。 -脚本加载本项目 `.env`,依赖 Java 21、Maven,以及正式执行时可用的 MySQL 和 Gitea。独立导入模块不启动 Web 服务,也不加载登录、AI 或 OSS 组件。Gitea 技术账号需具备组织所有者及仓库写权限,实例需开启迁移、HTTP Git 和 LFS。`GITHUB_TOKEN` 可选,仅用于 GitHub 元数据 API;API 限流时读取同一公开仓库的 GitHub 网页,交叉核验仓库 ID、规范地址、公开状态和默认分支,缺失或不一致则预检失败。网页和图片请求均不携带凭据。`GITEA_TOKEN` 仅访问配置的 Gitea,不能代替 `GITHUB_TOKEN`;切勿把 Token 放进 URL 或命令参数。 +脚本加载本项目 `.env`,依赖 Java 21、Maven,以及正式执行时可用的 MySQL 和 Gitea。独立导入模块不启动 Web 服务,也不加载登录、AI 或 OSS 客户端;只读取资源地址配置,不需要 OSS 凭据。test/prod 根目录与在线应用一致,历史公开地址使用 `OSS_LEGACY_PUBLIC_BASE_URLS`。Gitea 技术账号需具备组织所有者及仓库写权限,实例需开启迁移、HTTP Git 和 LFS。`GITHUB_TOKEN` 可选,仅用于 GitHub 元数据 API;API 限流时读取同一公开仓库的 GitHub 网页,交叉核验仓库 ID、规范地址、公开状态和默认分支,缺失或不一致则预检失败。网页和图片请求均不携带凭据。`GITEA_TOKEN` 仅访问配置的 Gitea,不能代替 `GITHUB_TOKEN`;切勿把 Token 放进 URL 或命令参数。 ```bash IMPORT_EXCEL='/绝对路径/项目.xlsx' diff --git a/community-api/src/main/resources/application.yml b/community-api/src/main/resources/application.yml index ec01f12..b650a8d 100644 --- a/community-api/src/main/resources/application.yml +++ b/community-api/src/main/resources/application.yml @@ -27,6 +27,7 @@ community: access-key-id: ${OSS_ACCESS_KEY_ID:} access-key-secret: ${OSS_ACCESS_KEY_SECRET:} public-base-url: ${OSS_PUBLIC_BASE_URL:https://linkerbot.oss-cn-beijing.aliyuncs.com} + legacy-public-base-urls: ${OSS_LEGACY_PUBLIC_BASE_URLS:} urdf: max-file-bytes: ${URDF_MAX_FILE_BYTES:52428800} max-total-bytes: ${URDF_MAX_TOTAL_BYTES:209715200} diff --git a/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java b/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java index ad86f8e..f47f8c7 100644 --- a/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java +++ b/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java @@ -21,6 +21,7 @@ import com.linkerbot.community.service.ProjectService; import com.linkerbot.community.service.ai.AiModelClient; import com.linkerbot.community.service.ai.AiProjectAssistant; import com.linkerbot.community.service.config.AiProperties; +import com.linkerbot.community.service.config.OssProperties; import com.linkerbot.community.service.config.ForkProperties; import com.linkerbot.community.service.gitea.GiteaClient.Change; import com.linkerbot.community.service.mapper.ProjectMapper; @@ -91,6 +92,7 @@ class AiSearchTest { @Autowired AiProjectAssistant assistant; @Autowired FakeGitSnapshotCopier snapshots; @Autowired ForkProperties forkProperties; + @Autowired OssProperties oss; private User user; private String token; @@ -462,6 +464,9 @@ class AiSearchTest { @Test void forksExplicitCodeWithCustomNameAndCopiesPublishedSnapshot() throws Exception { Project source = publishedRepository("轮式教学"); + String cover = "/project-covers/111111111111/20261019/a.png"; + source.setCoverUrl(cover); + projects.updateById(source); model.handler = (name, input) -> forkIntent(source.getProjectCode(), "我的学习副本"); List events = finish(start(Map.of("message", "Fork " + source.getProjectCode() + ",命名为我的学习副本"))); JsonNode data = events.getLast().path("data"); @@ -474,6 +479,8 @@ class AiSearchTest { assertThat(fork.path("publishStatus").asText()).isEqualTo("UNPUBLISHED"); assertThat(fork.has("id")).isFalse(); Project copied = projectService.owner(user.getId(), fork.path("projectCode").asText()); + assertThat(copied.getCoverUrl()).isEqualTo(cover); + assertThat(fork.path("coverUrl").asText()).isEqualTo(oss.publicBaseUrl() + "/" + oss.getRootDirectory() + cover); assertThat(gitea.file(projectService.repository(copied), gitea.head(projectService.repository(copied)), "README.md").bytes()) .isEqualTo(gitea.file(projectService.repository(source), source.getPublishedCommitSha(), "README.md").bytes()); assertThat(projects.selectCount(null)).isEqualTo(2L); @@ -483,6 +490,23 @@ class AiSearchTest { .containsExactly("understanding", "forking"); } + @Test void createsHostedCoverAndSearchReturnsPublicUrlForStoredPath() throws Exception { + String cover = "/project-covers/111111111111/20261019/a.png"; + String fullUrl = oss.publicBaseUrl() + "/" + oss.getRootDirectory() + cover; + model.handler = (name, input) -> createIntent("WHEELED", "路径测试机器人").put("coverUrl", fullUrl); + JsonNode created = finish(start(Map.of("message", "创建路径测试机器人,使用给定封面"))) + .getLast().path("data").path("project"); + assertThat(created.path("coverUrl").asText()).isEqualTo(fullUrl); + Project project = projectService.owner(user.getId(), created.path("projectCode").asText()); + assertThat(project.getCoverUrl()).isEqualTo(cover); + projectService.publish(user.getId(), project.getProjectCode(), gitea.head(projectService.repository(project))); + model.handler = (name, input) -> name.endsWith("intent") ? intent("WHEELED", List.of("路径测试")) : rank(input); + JsonNode card = finish(start(Map.of("message", "寻找路径测试机器人"))) + .getLast().path("data").path("projects").get(0).path("project"); + assertThat(card.path("coverUrl").asText()).isEqualTo(fullUrl); + assertThat(projectService.owner(user.getId(), project.getProjectCode()).getCoverUrl()).isEqualTo(cover); + } + @Test void forkGetsItsOwnDeadlineAfterUnderstandingRatherThanTheSearchTimeout() throws Exception { Project source = publishedRepository("教学项目"); properties.setSearchTimeout(Duration.ofMillis(150)); diff --git a/community-api/src/test/java/com/linkerbot/community/api/ResourcePathMigrationTest.java b/community-api/src/test/java/com/linkerbot/community/api/ResourcePathMigrationTest.java new file mode 100644 index 0000000..b688eba --- /dev/null +++ b/community-api/src/test/java/com/linkerbot/community/api/ResourcePathMigrationTest.java @@ -0,0 +1,150 @@ +package com.linkerbot.community.api; + +import static org.assertj.core.api.Assertions.*; + +import com.linkerbot.community.service.config.OssProperties; +import com.linkerbot.community.service.oss.OssResourcePaths; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.SingleConnectionDataSource; +import org.springframework.jdbc.datasource.DriverManagerDataSource; + +/** 执行数据库文档中的 SQL;仅将 MySQL JOIN UPDATE、排序规则和正则语法适配到 H2。 */ +class ResourcePathMigrationTest { + private static final String ROOT = "vibemake_dev"; + private static final String BASE = "https://linkerbot.oss-cn-beijing.aliyuncs.com"; + private static final String CODE = "1".repeat(12); + private static final String COVER = "/project-covers/" + "1".repeat(32) + "/20261019/" + "a".repeat(32) + ".png"; + private static final String PREFIX = "/urdf/" + CODE + "/" + "a".repeat(32) + "/"; + private static final String MAIN = PREFIX + "urdf/机械 臂 #?%+.URDF"; + private SingleConnectionDataSource source; + private JdbcTemplate jdbc; + private String document; + private OssResourcePaths paths; + + @BeforeEach + void prepare() throws Exception { + source = new SingleConnectionDataSource("jdbc:h2:mem:migration;MODE=MySQL", "sa", "", true); + jdbc = new JdbcTemplate(source); + jdbc.execute("CREATE TABLE project (project_code VARCHAR(32) PRIMARY KEY, cover_url VARCHAR(2048)," + + " urdf_main_object_key VARCHAR(512) NOT NULL, urdf_prefix VARCHAR(512) NOT NULL)"); + document = Files.readString(Path.of("../docs/database.md")); + OssProperties oss = new OssProperties(); + oss.setRootDirectory(ROOT); + oss.setPublicBaseUrl(BASE); + paths = new OssResourcePaths(oss); + } + + @AfterEach + void close() { + source.destroy(); + } + + @Test + void migrationIsIdempotentAndPreservesExternalUrlsAndObjectLocations() { + insert(CODE, BASE + "/" + ROOT + COVER, ROOT + MAIN, ROOT + PREFIX); + insert("2".repeat(12), "https://external.example.com" + COVER, "", ""); + insert("3".repeat(12), BASE + "/" + ROOT + COVER + "?version=2", "", ""); + insert("4".repeat(12), BASE + "/" + ROOT + COVER + "#preview", "", ""); + insert("5".repeat(12), COVER, "/urdf/already/a.urdf", "/urdf/already/"); + insert("6".repeat(12), BASE + "/vibemake" + COVER, "vibemake" + MAIN, "vibemake" + PREFIX); + insert("7".repeat(12), null, ROOT + "/urdf/wrong/a.urdf", ROOT + PREFIX); + insert("8".repeat(12), BASE + ".evil/" + ROOT + COVER, "", ""); + insert("9".repeat(12), BASE + "/VIBEMAKE_DEV" + COVER, "", ""); + Map externalBefore = row("2".repeat(12)); + String publicCover = paths.coverUrl(row(CODE).get("COVER_URL").toString()); + String publicMain = paths.publicUrl(ROOT + MAIN); + execute("migration-prepare"); + assertThat(jdbc.queryForObject("SELECT COUNT(*) FROM resource_path_changes", Integer.class)).isEqualTo(1); + execute("migration-apply"); + assertThat(row(CODE)).containsEntry("COVER_URL", COVER) + .containsEntry("URDF_MAIN_OBJECT_KEY", MAIN).containsEntry("URDF_PREFIX", PREFIX); + assertThat(paths.coverUrl(row(CODE).get("COVER_URL").toString())).isEqualTo(publicCover); + assertThat(paths.publicUrl(row(CODE).get("URDF_MAIN_OBJECT_KEY").toString())).isEqualTo(publicMain); + assertThat(row("2".repeat(12))).isEqualTo(externalBefore); + assertThat(row("3".repeat(12)).get("COVER_URL")).asString().endsWith("?version=2"); + assertThat(row("4".repeat(12)).get("COVER_URL")).asString().endsWith("#preview"); + assertThat(row("6".repeat(12)).get("URDF_PREFIX")).isEqualTo("vibemake" + PREFIX); + assertThat(row("7".repeat(12)).get("URDF_PREFIX")).isEqualTo(ROOT + PREFIX); + assertThat(row("8".repeat(12)).get("COVER_URL")).asString().contains(".evil/"); + assertThat(row("9".repeat(12)).get("COVER_URL")).asString().contains("/VIBEMAKE_DEV/"); + execute("migration-prepare"); + assertThat(jdbc.queryForObject("SELECT COUNT(*) FROM resource_path_changes", Integer.class)).isZero(); + execute("migration-apply"); + assertThat(row(CODE).get("COVER_URL")).isEqualTo(COVER); + } + + @Test + void concurrentChangesAreNotOverwrittenAndNewPathsCanBeRolledBack() { + insert(CODE, BASE + "/" + ROOT + COVER, ROOT + MAIN, ROOT + PREFIX); + execute("migration-prepare"); + jdbc.update("UPDATE project SET cover_url = ? WHERE project_code = ?", "https://external.example.com/new.png", CODE); + execute("migration-apply"); + assertThat(row(CODE)).containsEntry("COVER_URL", "https://external.example.com/new.png") + .containsEntry("URDF_MAIN_OBJECT_KEY", ROOT + MAIN); + jdbc.update("UPDATE project SET cover_url = ? WHERE project_code = ?", BASE + "/" + ROOT + COVER, CODE); + execute("migration-prepare"); + execute("migration-apply"); + execute("rollback-prepare"); + jdbc.update("INSERT INTO resource_cover_rollback VALUES (?, ?, ?)", CODE, COVER, paths.coverUrl(COVER)); + execute("rollback-apply"); + assertThat(row(CODE)).containsEntry("COVER_URL", BASE + "/" + ROOT + COVER) + .containsEntry("URDF_MAIN_OBJECT_KEY", ROOT + MAIN).containsEntry("URDF_PREFIX", ROOT + PREFIX); + execute("rollback-apply"); + assertThat(row(CODE).get("URDF_PREFIX")).isEqualTo(ROOT + PREFIX); + } + + private void insert(String code, String cover, String main, String prefix) { + jdbc.update("INSERT INTO project VALUES (?, ?, ?, ?)", code, cover, main, prefix); + } + + @Test + void urdfResumeCommitsEarlierCoverChangeWithoutStartingAnotherTransaction() throws Exception { + insert(CODE, BASE + "/" + ROOT + COVER, ROOT + MAIN, ROOT + PREFIX); + source.getConnection().setAutoCommit(false); + jdbc.update("UPDATE project SET cover_url = ? WHERE project_code = ?", COVER, CODE); + execute("migration-resume-urdf"); + JdbcTemplate observer = new JdbcTemplate( + new DriverManagerDataSource("jdbc:h2:mem:migration;MODE=MySQL", "sa", "")); + assertThat(observer.queryForMap("SELECT * FROM project WHERE project_code = ?", CODE)) + .containsEntry("COVER_URL", COVER).containsEntry("URDF_MAIN_OBJECT_KEY", MAIN) + .containsEntry("URDF_PREFIX", PREFIX); + execute("migration-resume-urdf"); + assertThat(row(CODE).get("URDF_PREFIX")).isEqualTo(PREFIX); + } + + private Map row(String code) { + return jdbc.queryForMap("SELECT * FROM project WHERE project_code = ?", code); + } + + private void execute(String name) { + Matcher block = Pattern.compile("```sql\\s*-- resource-path-" + name + "\\s+([\\s\\S]*?)```").matcher(document); + assertThat(block.find()).as("文档 SQL 块 %s", name).isTrue(); + String sql = block.group(1).replaceAll("(?m)^--.*$", "") + .replace(" COLLATE utf8mb4_bin", "") + .replace(" CHARACTER SET utf8mb4", "") + .replace("DROP TEMPORARY TABLE", "DROP TABLE") + .replace("START TRANSACTION", "SET AUTOCOMMIT FALSE") + .replace("SET NAMES utf8mb4", "") + .replace("<=>", "IS NOT DISTINCT FROM") + .replace("[[:cntrl:]:]", "[\\p{Cntrl}:]"); + for (String statement : sql.split(";")) { + if (statement.isBlank()) continue; + Matcher update = Pattern.compile("UPDATE project p JOIN (\\w+) c ON c.project_code = p.project_code\\s+SET ([\\s\\S]*?)\\s+WHERE ([\\s\\S]*)") + .matcher(statement.trim()); + if (update.matches()) { + statement = "MERGE INTO project p USING " + update.group(1) + + " c ON c.project_code = p.project_code WHEN MATCHED AND " + update.group(3) + + " THEN UPDATE SET " + update.group(2).replace("p.", ""); + } + jdbc.execute(statement); + } + } +} diff --git a/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java b/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java index 5f41bd1..edb34cf 100644 --- a/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java +++ b/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java @@ -14,6 +14,8 @@ import com.linkerbot.community.service.config.UrdfProperties; import com.linkerbot.community.common.BusinessException; import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.autoconfigure.web.servlet.MockMvcPrint; @@ -84,6 +86,7 @@ class WorkflowTest { snapshots.available = true; snapshots.beforePush = null; oss.setEnabled(false); + oss.setLegacyPublicBaseUrls(List.of()); ossObjects.reset(); urdfLimits.setMaxFileBytes(52428800); urdfLimits.setMaxTotalBytes(209715200); @@ -358,6 +361,41 @@ class WorkflowTest { .path("data").path("coverUrl").asText()).isEqualTo(max); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void hostedCoversStoreRelativePathsAndLegacyUrlsRemainUsableByFork(boolean legacy) throws Exception { + enableUrdfOss(); + String owner = login("hosted-cover-owner@example.com"); + String other = login("hosted-cover-other@example.com"); + String stored = "/project-covers/111111111111/20261019/a.png"; + String uploaded = oss.uploadUrl() + "/vibemake_dev" + stored; + String expected = "https://images.example.com/assets/vibemake_dev" + stored; + Map input = new LinkedHashMap<>(Map.of( + "name", "平台封面", "projectType", "GENERAL", "coverUrl", uploaded)); + JsonNode created = perform(body(auth(post("/api/workspace/projects"), owner), input), 200).path("data"); + String code = created.path("projectCode").asText(); + String endpoint = "/api/workspace/projects/" + code; + assertThat(created.path("coverUrl").asText()).isEqualTo(expected); + assertThat(jdbc.queryForObject("SELECT cover_url FROM project WHERE project_code = ?", String.class, code)).isEqualTo(stored); + if (legacy) jdbc.update("UPDATE project SET cover_url = ? WHERE project_code = ?", uploaded, code); + assertThat(perform(auth(get(endpoint), owner), 200).path("data").path("coverUrl").asText()).isEqualTo(expected); + assertThat(jdbc.queryForObject("SELECT cover_url FROM project WHERE project_code = ?", String.class, code)) + .isEqualTo(legacy ? uploaded : stored); // 读取不会隐式迁移。 + String sha = head(owner, code); + publish(owner, code, sha); + JsonNode fork = perform(body(auth(post("/api/community/projects/" + code + "/forks"), other), + Map.of("name", "封面副本", "publishedCommitSha", sha)), 200).path("data"); + assertThat(fork.path("coverUrl").asText()).isEqualTo(expected); + assertThat(jdbc.queryForObject("SELECT cover_url FROM project WHERE project_code = ?", String.class, + fork.path("projectCode").asText())).isEqualTo(stored); + input.put("coverUrl", expected); + perform(body(auth(patch(endpoint), owner), input), 200); + assertThat(jdbc.queryForObject("SELECT cover_url FROM project WHERE project_code = ?", String.class, code)).isEqualTo(stored); + oss.setPublicBaseUrl("https://new-cdn.example.com/base"); + assertThat(perform(get("/api/community/projects/" + code), 200).path("data").path("coverUrl").asText()) + .isEqualTo("https://new-cdn.example.com/base/vibemake_dev" + stored); + } + private void enableUrdfOss() { oss.setRegion("cn-hangzhou"); oss.setEndpoint(""); @@ -463,7 +501,9 @@ class WorkflowTest { assertThat(bound.path("urdfMainUrl").asText()).isEqualTo(baseUrl + "urdf/%E6%9C%BA%E6%A2%B0%20%E8%87%82.URDF"); assertThat(bound.has("id")).isFalse(); assertThat(jdbc.queryForObject("SELECT urdf_main_object_key FROM project WHERE project_code = ?", String.class, code)) - .isEqualTo(expectedPrefix + "urdf/机械 臂.URDF"); + .isEqualTo(expectedPrefix.substring("vibemake_dev".length()) + "urdf/机械 臂.URDF"); + assertThat(jdbc.queryForObject("SELECT urdf_prefix FROM project WHERE project_code = ?", String.class, code)) + .isEqualTo(expectedPrefix.substring("vibemake_dev".length())); assertThat(json.readTree(jdbc.queryForObject("SELECT urdf_files FROM project WHERE project_code = ?", String.class, code))) .isEqualTo(json.valueToTree(files)); assertThat(perform(get("/api/community/projects/" + code), 200).path("data").path("urdfMainUrl")).isEqualTo(bound.path("urdfMainUrl")); @@ -510,8 +550,9 @@ class WorkflowTest { .contains(first.path("batchCode").asText()); } - @Test - void urdfForkCopiesIndependentObjectsAndCopyFailureDoesNotReturnReady() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void urdfForkCopiesIndependentObjectsAndCopyFailureDoesNotReturnReady(boolean legacy) throws Exception { String owner = login("urdf-fork-source@example.com"); String other = login("urdf-fork-target@example.com"); String code = create(owner, "URDF 来源"); @@ -519,6 +560,8 @@ class WorkflowTest { List> files = urdfFiles(); JsonNode uploaded = uploadUrdf(owner, code, files); JsonNode source = confirmUrdf(owner, code, uploaded, files, 200).path("data"); + if (legacy) jdbc.update("UPDATE project SET urdf_prefix = CONCAT('vibemake_dev', urdf_prefix)," + + " urdf_main_object_key = CONCAT('vibemake_dev', urdf_main_object_key) WHERE project_code = ?", code); String sha = head(owner, code); publish(owner, code, sha); String forkEndpoint = "/api/community/projects/" + code + "/forks"; @@ -528,8 +571,9 @@ class WorkflowTest { assertThat(fork.path("repoStatus").asText()).isEqualTo("READY"); assertThat(fork.path("urdfMainUrl").asText()).contains("/urdf/" + targetCode + "/").doesNotContain("/urdf/" + code + "/"); String targetPrefix = jdbc.queryForObject("SELECT urdf_prefix FROM project WHERE project_code = ?", String.class, targetCode); + assertThat(targetPrefix).startsWith("/urdf/" + targetCode + "/"); for (int index = 0; index < files.size(); index++) - assertThat(ossObjects.objects.get(targetPrefix + files.get(index).get("relativePath"))) + assertThat(ossObjects.objects.get("vibemake_dev" + targetPrefix + files.get(index).get("relativePath"))) .containsExactly(ossObjects.objects.get(uploaded.path("files").get(index).path("objectKey").asText())); confirmUrdf(owner, code, uploadUrdf(owner, code, files), files, 200); perform(auth(delete("/api/workspace/projects/" + code + "/urdf"), owner), 200); diff --git a/community-import/src/main/java/com/linkerbot/community/importer/DatabaseConfiguration.java b/community-import/src/main/java/com/linkerbot/community/importer/DatabaseConfiguration.java index 4ef8d88..35230b9 100644 --- a/community-import/src/main/java/com/linkerbot/community/importer/DatabaseConfiguration.java +++ b/community-import/src/main/java/com/linkerbot/community/importer/DatabaseConfiguration.java @@ -2,6 +2,8 @@ package com.linkerbot.community.importer; import com.linkerbot.community.service.config.CommunityProperties; import com.linkerbot.community.service.config.MybatisConfiguration; +import com.linkerbot.community.service.config.OssProperties; +import com.linkerbot.community.service.oss.OssResourcePaths; import com.linkerbot.community.service.gitea.RestGiteaClient; import com.linkerbot.community.service.impl.ProjectServiceImpl; import com.linkerbot.community.service.impl.UserServiceImpl; @@ -12,7 +14,7 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -/** 显式加载所需组件,避免扫描在线应用的安全、AI 和 OSS 模块。 */ +/** 仅加载资源路径配置,不初始化在线安全、AI 或 OSS 客户端。 */ @Configuration(proxyBeanMethods = false) @EnableAutoConfiguration @MapperScan("com.linkerbot.community.service.mapper") @@ -22,6 +24,8 @@ import org.springframework.context.annotation.Import; RestGiteaClient.class, ProjectServiceImpl.class, UserServiceImpl.class, - ProjectLocks.class + ProjectLocks.class, + OssProperties.class, + OssResourcePaths.class }) public class DatabaseConfiguration {} diff --git a/community-import/src/main/java/com/linkerbot/community/importer/ImportApplication.java b/community-import/src/main/java/com/linkerbot/community/importer/ImportApplication.java index 657e3ef..71343af 100644 --- a/community-import/src/main/java/com/linkerbot/community/importer/ImportApplication.java +++ b/community-import/src/main/java/com/linkerbot/community/importer/ImportApplication.java @@ -8,6 +8,7 @@ import com.linkerbot.community.bean.entity.User; import com.linkerbot.community.common.BusinessException; import com.linkerbot.community.common.ProjectCodes; import com.linkerbot.community.service.ProjectService; +import com.linkerbot.community.service.oss.OssResourcePaths; import com.linkerbot.community.service.config.CommunityProperties; import com.linkerbot.community.service.gitea.*; import com.linkerbot.community.service.mapper.ProjectMapper; @@ -90,6 +91,12 @@ public final class ImportApplication { } public static void main(String[] args) { + if (args.length > 0 && "covers".equals(args[0])) { + int status = com.linkerbot.community.importer.cover.CoverMigrationApplication.runCli( + Arrays.copyOfRange(args, 1, args.length)); + if (status != 0) System.exit(status); + return; + } if (args.length == 0 || Arrays.asList(args).contains("--help")) { System.out.println( "用法:--excel <文件> [--state-dir <目录>] [--dry-run | --execute (--owner-user-code" @@ -224,6 +231,7 @@ public final class ImportApplication { context.getBean(ProjectService.class), context.getBean(GiteaClient.class), context.getBean(CommunityProperties.class), + context.getBean(OssResourcePaths.class), store, owner, options.migrationTimeout()); diff --git a/community-import/src/main/java/com/linkerbot/community/importer/ImportCoordinator.java b/community-import/src/main/java/com/linkerbot/community/importer/ImportCoordinator.java index bda611f..f98616f 100644 --- a/community-import/src/main/java/com/linkerbot/community/importer/ImportCoordinator.java +++ b/community-import/src/main/java/com/linkerbot/community/importer/ImportCoordinator.java @@ -8,6 +8,7 @@ import com.linkerbot.community.bean.entity.User; import com.linkerbot.community.common.BusinessException; import com.linkerbot.community.service.ProjectService; import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.oss.OssResourcePaths; import com.linkerbot.community.service.gitea.*; import com.linkerbot.community.service.gitea.GiteaClient.RemoteRepository; import com.linkerbot.community.service.mapper.ProjectMapper; @@ -27,6 +28,7 @@ public final class ImportCoordinator { private final ProjectService service; private final GiteaClient gitea; private final CommunityProperties config; + private final OssResourcePaths resourcePaths; private final StateStore store; private final User owner; private final Duration timeout; @@ -46,6 +48,7 @@ public final class ImportCoordinator { ProjectService service, GiteaClient gitea, CommunityProperties config, + OssResourcePaths resourcePaths, StateStore store, User owner, Duration timeout) { @@ -53,6 +56,7 @@ public final class ImportCoordinator { this.service = service; this.gitea = gitea; this.config = config; + this.resourcePaths = resourcePaths; this.store = store; this.owner = owner; this.timeout = timeout; @@ -97,7 +101,7 @@ public final class ImportCoordinator { project.setOwnerUserId(owner.getId()); project.setName(entry.candidate.name().trim()); project.setDescription(entry.candidate.description()); - project.setCoverUrl(ProjectInputs.coverUrl(entry.candidate.coverUrl())); + project.setCoverUrl(resourcePaths.coverStorage(entry.candidate.coverUrl())); project.setProjectType(entry.candidate.projectType()); project.setGiteaOwner(config.getGitea().getOwner()); project.setGiteaRepoName(repoName); diff --git a/community-import/src/main/java/com/linkerbot/community/importer/cover/CoverMigrationApplication.java b/community-import/src/main/java/com/linkerbot/community/importer/cover/CoverMigrationApplication.java new file mode 100644 index 0000000..b9425c7 --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/cover/CoverMigrationApplication.java @@ -0,0 +1,172 @@ +package com.linkerbot.community.importer.cover; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.linkerbot.community.service.config.OssProperties; +import com.linkerbot.community.service.oss.OssResourcePaths; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.Instant; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.UUID; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.jdbc.core.JdbcTemplate; + +public final class CoverMigrationApplication { + @Configuration(proxyBeanMethods = false) + @EnableAutoConfiguration + @Import({OssProperties.class, OssResourcePaths.class}) + public static class Config {} + + public record Options(boolean execute, int limit, int concurrency, int maxBytes, Path reportDirectory, Path sourceOverrides) { + public static Options parse(String[] args) { + boolean execute = false; + boolean dryRun = false; + int limit = Integer.MAX_VALUE; + int concurrency = 2; + int maxBytes = 5 * 1024 * 1024; + Path directory = Path.of("logs/cover-migration"); + Path overrides = null; + List seen = new ArrayList<>(); + for (int index = 0; index < args.length; index++) { + String key = args[index]; + if (seen.contains(key)) throw new IllegalArgumentException("参数重复"); + seen.add(key); + if ("--execute".equals(key)) execute = true; + else if ("--dry-run".equals(key)) dryRun = true; + else { + if (index + 1 == args.length) throw new IllegalArgumentException("缺少参数值"); + String value = args[++index]; + switch (key) { + case "--limit" -> limit = Integer.parseInt(value); + case "--concurrency" -> concurrency = Integer.parseInt(value); + case "--max-bytes" -> maxBytes = Integer.parseInt(value); + case "--report-dir" -> directory = Path.of(value); + case "--source-overrides" -> overrides = Path.of(value); + default -> throw new IllegalArgumentException("不支持的参数"); + } + } + } + if (execute && dryRun || limit < 1 || concurrency < 1 || concurrency > 4 || maxBytes < 1 || maxBytes > 50 * 1024 * 1024) + throw new IllegalArgumentException("参数范围不合法"); + return new Options(execute, limit, concurrency, maxBytes, directory, overrides); + } + } + + public static int runCli(String[] args) { + if (Arrays.asList(args).contains("--help")) { + System.out.println("用法:covers [--dry-run | --execute] [--limit N] [--concurrency 2]" + + " [--max-bytes 5242880] [--report-dir logs/cover-migration] [--source-overrides ]"); + return 0; + } + try { + Options options = Options.parse(args); + try (ConfigurableApplicationContext context = new SpringApplicationBuilder(Config.class) + .web(WebApplicationType.NONE).logStartupInfo(false).run()) { + return run(options, context.getBean(JdbcTemplate.class), context.getBean(OssProperties.class), + context.getBean(OssResourcePaths.class)); + } + } catch (Exception error) { + String code = error instanceof ExternalCoverDownloader.Failure failure ? failure.code() : error.getClass().getSimpleName(); + System.err.println("封面迁移终止 code=" + code + ";检查本地配置和运行报告,不输出凭据或原始异常文本"); + for (Throwable cause = error; cause != null; cause = cause.getCause()) { + if (cause instanceof SQLException sql) + System.err.printf("数据库异常 sqlState=%s vendorCode=%d type=%s%n", sql.getSQLState(), sql.getErrorCode(), sql.getClass().getSimpleName()); + } + return 1; + } + } + + static int run(Options options, JdbcTemplate jdbc, OssProperties oss, OssResourcePaths paths) throws Exception { + CoverMigrator.Repository repository = new CoverMigrator.JdbcRepository(jdbc); + ExternalCoverDownloader downloader = new ExternalCoverDownloader(options.maxBytes()); + CoverMigrator scanner = new CoverMigrator(repository, paths, downloader::download, (key, image, sha) -> {}); + List candidates = scanner.candidates(options.limit()); + long unique = candidates.stream().map(candidate -> candidate.originalUrl().trim()).distinct().count(); + System.out.printf("封面迁移预览 environmentRoot=%s candidateProjects=%d uniqueUrls=%d execute=%s%n", + oss.getRootDirectory(), candidates.size(), unique, options.execute()); + Path run = options.reportDirectory().resolve(Instant.now().toString().replace(':', '-') + "-" + UUID.randomUUID().toString().substring(0, 8)); + Files.createDirectories(run); + restrict(run, "rwx------"); + ObjectMapper json = new ObjectMapper(); + Map overrides = options.sourceOverrides() == null ? Map.of() + : json.readValue(Files.readString(options.sourceOverrides()), new TypeReference>() {}); + Map sources = downloadSources(candidates, overrides); + List> preview = candidates.stream().map(candidate -> Map.of( + "projectCode", candidate.projectCode(), "sourceHost", String.valueOf(CoverMigrator.host(candidate.originalUrl())))).toList(); + writeReport(json, run.resolve("preview.json"), preview); + System.out.println("迁移报告目录=" + run.toAbsolutePath()); + if (!options.execute() || candidates.isEmpty()) return 0; + if (!sources.isEmpty()) writeReport(json, run.resolve("source-overrides-backup.json"), overrides); + try (CoverOssUploader uploader = new CoverOssUploader(oss); + DurableBackup backup = new DurableBackup(run.resolve("backup.jsonl"), json)) { + CoverMigrator migrator = new CoverMigrator(repository, paths, + url -> downloader.download(sources.getOrDefault(url, url)), uploader); + List results = migrator.migrate(candidates, options.concurrency(), backup::record); + writeReport(json, run.resolve("report.json"), results); + long updated = results.stream().filter(result -> "UPDATED".equals(result.status())).count(); + long failed = results.stream().filter(result -> "FAILED".equals(result.status())).count(); + long changed = results.size() - updated - failed; + System.out.printf("封面迁移结束 updated=%d failed=%d changedSkipped=%d%n", updated, failed, changed); + return failed == 0 ? 0 : 2; + } + } + + static Map downloadSources(List candidates, Map overrides) { + Map sources = new LinkedHashMap<>(); + for (CoverMigrator.Candidate candidate : candidates) { + String source = overrides.get(candidate.projectCode()); + if (source == null) continue; + source = com.linkerbot.community.service.support.ProjectInputs.coverUrl(source); + if (source == null) throw new IllegalArgumentException("备用下载地址不能为空"); + String previous = sources.putIfAbsent(candidate.originalUrl().trim(), source); + if (previous != null && !previous.equals(source)) + throw new IllegalArgumentException("相同原始封面不能配置不同下载源"); + } + return Map.copyOf(sources); + } + + private static void writeReport(ObjectMapper json, Path path, Object value) throws Exception { + Files.write(path, json.writerWithDefaultPrettyPrinter().writeValueAsBytes(value), StandardOpenOption.CREATE_NEW); + restrict(path, "rw-------"); + } + + static void restrict(Path path, String mode) throws Exception { + if (Files.getFileStore(path).supportsFileAttributeView("posix")) + Files.setPosixFilePermissions(path, PosixFilePermissions.fromString(mode)); + } + + static final class DurableBackup implements AutoCloseable { + private final FileChannel channel; + private final ObjectMapper json; + DurableBackup(Path path, ObjectMapper json) throws Exception { + this.json = json; + channel = FileChannel.open(path, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); + restrict(path, "rw-------"); + } + synchronized void record(CoverMigrator.Candidate candidate, String path) throws Exception { + // 原始地址仅保存于受限备份,不进入普通日志;先持久化再修改数据库。 + byte[] bytes = (json.writeValueAsString(Map.of("projectCode", candidate.projectCode(), + "originalCoverUrl", candidate.originalUrl(), "newCoverPath", path)) + "\n").getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.wrap(bytes); + while (buffer.hasRemaining()) channel.write(buffer); + channel.force(true); + } + @Override public void close() throws Exception { channel.close(); } + } +} diff --git a/community-import/src/main/java/com/linkerbot/community/importer/cover/CoverMigrator.java b/community-import/src/main/java/com/linkerbot/community/importer/cover/CoverMigrator.java new file mode 100644 index 0000000..4d07434 --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/cover/CoverMigrator.java @@ -0,0 +1,137 @@ +package com.linkerbot.community.importer.cover; + +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.oss.OssResourcePaths; +import java.net.URI; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; + +public class CoverMigrator { + public record Candidate(String projectCode, String originalUrl) {} + public record Result(String projectCode, String sourceHost, String status, String code, String coverPath) {} + public interface Repository { + List list(); + boolean replace(Candidate candidate, String coverPath); + } + @FunctionalInterface public interface Downloader { ExternalCoverDownloader.Image download(String url) throws Exception; } + @FunctionalInterface public interface Uploader { void upload(String key, ExternalCoverDownloader.Image image, String sha256) throws Exception; } + @FunctionalInterface public interface Backup { void record(Candidate candidate, String path) throws Exception; } + + public static class JdbcRepository implements Repository { + private final JdbcTemplate jdbc; + public JdbcRepository(JdbcTemplate jdbc) { this.jdbc = jdbc; } + @Override public List list() { + return jdbc.query("SELECT project_code, cover_url FROM project WHERE cover_url IS NOT NULL" + + " AND cover_url <> '' ORDER BY project_code", + (row, index) -> new Candidate(row.getString("project_code"), row.getString("cover_url"))); + } + @Override public boolean replace(Candidate candidate, String path) { + // HEX 比较原始 UTF-8 字节,避免默认不区分大小写的排序规则覆盖并发修改。 + return jdbc.update("UPDATE project SET cover_url = ? WHERE project_code = ? AND HEX(cover_url) = HEX(?)", + path, candidate.projectCode(), candidate.originalUrl()) == 1; + } + } + + private static final Logger log = LoggerFactory.getLogger(CoverMigrator.class); + private final Repository repository; + private final OssResourcePaths paths; + private final Downloader downloader; + private final Uploader uploader; + + public CoverMigrator(Repository repository, OssResourcePaths paths, Downloader downloader, Uploader uploader) { + this.repository = repository; + this.paths = paths; + this.downloader = downloader; + this.uploader = uploader; + } + + public List candidates(int limit) { + List candidates = new ArrayList<>(); + for (Candidate candidate : repository.list()) { + String value = candidate.originalUrl(); + if (value == null || value.isBlank() || value.startsWith("/")) continue; + try { + String normalized = paths.coverStorage(value); + if (normalized != null && normalized.startsWith("/project-covers/")) continue; + } catch (BusinessException | IllegalArgumentException error) { + // 非法历史外链进入失败报告,不因单条坏数据终止整个批次。 + } + candidates.add(candidate); + if (candidates.size() == limit) break; + } + return List.copyOf(candidates); + } + + public List migrate(List candidates, int concurrency, Backup backup) throws Exception { + Map> groups = new LinkedHashMap<>(); + for (Candidate candidate : candidates) + groups.computeIfAbsent(candidate.originalUrl().trim(), ignored -> new ArrayList<>()).add(candidate); + List results = Collections.synchronizedList(new ArrayList<>()); + try (ExecutorService executor = Executors.newFixedThreadPool(concurrency)) { + List> jobs = new ArrayList<>(); + for (Map.Entry> group : groups.entrySet()) + jobs.add(executor.submit(() -> migrateGroup(group.getKey(), group.getValue(), backup, results))); + for (Future job : jobs) job.get(); + } + results.sort(Comparator.comparing(Result::projectCode)); + return List.copyOf(results); + } + + private void migrateGroup(String url, List group, Backup backup, List results) { + String phase = "DOWNLOAD"; + String path; + try { + ExternalCoverDownloader.Image image = downloader.download(url); + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(image.bytes())); + path = "/project-covers/migrated/" + digest + "." + image.extension(); + phase = "UPLOAD"; + uploader.upload(paths.objectKey(path), image, digest); + } catch (Exception error) { + for (Candidate candidate : group) failed(candidate, phase, error, results); + return; + } + for (Candidate candidate : group) { + phase = "BACKUP"; + try { + backup.record(candidate, path); + phase = "UPDATE"; + boolean updated = repository.replace(candidate, path); + String status = updated ? "UPDATED" : "CHANGED"; + results.add(new Result(candidate.projectCode(), host(url), status, "", path)); + log.info("封面转存完成 projectCode={} status={} coverPath={}", candidate.projectCode(), status, path); + } catch (Exception error) { + failed(candidate, phase, error, results); + } + } + } + + private void failed(Candidate candidate, String phase, Exception error, List results) { + String code = error instanceof ExternalCoverDownloader.Failure failure ? failure.code() + : error instanceof BusinessException business ? business.code() : error.getClass().getSimpleName(); + if (error.getCause() != null) code += "/" + error.getCause().getClass().getSimpleName(); + results.add(new Result(candidate.projectCode(), host(candidate.originalUrl()), "FAILED", phase + ":" + code, null)); + // 异常文本可能包含带参数的原始 URL,不打印原始异常及查询参数。 + log.warn("封面转存失败 projectCode={} sourceHost={} phase={} code={}", + candidate.projectCode(), host(candidate.originalUrl()), phase, code); + } + + public static String host(String url) { + try { + String host = URI.create(url.trim()).getHost(); + return host == null ? "invalid" : host; + } + catch (IllegalArgumentException error) { return "invalid"; } + } +} diff --git a/community-import/src/main/java/com/linkerbot/community/importer/cover/CoverOssUploader.java b/community-import/src/main/java/com/linkerbot/community/importer/cover/CoverOssUploader.java new file mode 100644 index 0000000..5aaf584 --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/cover/CoverOssUploader.java @@ -0,0 +1,72 @@ +package com.linkerbot.community.importer.cover; + +import com.aliyun.oss.ClientBuilderConfiguration; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.common.auth.DefaultCredentialProvider; +import com.aliyun.oss.common.comm.SignVersion; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.PutObjectRequest; +import com.linkerbot.community.service.config.OssProperties; +import java.io.ByteArrayInputStream; +import java.util.Map; + +final class CoverOssUploader implements AutoCloseable, CoverMigrator.Uploader { + private final OSS client; + private final String bucket; + + CoverOssUploader(OssProperties properties) { + properties.setEnabled(true); + if (!properties.isConfigurationValid()) throw new ExternalCoverDownloader.Failure("OSS_CONFIGURATION_MISSING"); + ClientBuilderConfiguration configuration = new ClientBuilderConfiguration(); + configuration.setSignatureVersion(SignVersion.V4); + configuration.setConnectionTimeout(10000); + configuration.setSocketTimeout(20000); + configuration.setRequestTimeoutEnabled(true); + configuration.setRequestTimeout(30000); + configuration.setMaxErrorRetry(1); + client = OSSClientBuilder.create().endpoint(properties.resolvedEndpoint()).region(properties.getRegion()) + .credentialsProvider(new DefaultCredentialProvider(properties.getAccessKeyId(), properties.getAccessKeySecret())) + .clientConfiguration(configuration).build(); + bucket = properties.getBucket(); + } + + @Override + public void upload(String key, ExternalCoverDownloader.Image image, String sha256) { + ObjectMetadata existing = metadata(key); + if (existing != null) { + verify(existing, image, sha256); + return; + } + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(image.bytes().length); + metadata.setContentType(image.contentType()); + metadata.setUserMetadata(Map.of("sha256", sha256)); + metadata.setHeader("x-oss-forbid-overwrite", "true"); + try { + client.putObject(new PutObjectRequest(bucket, key, new ByteArrayInputStream(image.bytes()), metadata)); + } catch (OSSException error) { + if (!"FileAlreadyExists".equals(error.getErrorCode())) + throw new ExternalCoverDownloader.Failure("OSS_PUT_" + error.getErrorCode()); + } + ObjectMetadata uploaded = metadata(key); + if (uploaded == null) throw new ExternalCoverDownloader.Failure("OSS_VERIFY_MISSING"); + verify(uploaded, image, sha256); + } + + private ObjectMetadata metadata(String key) { + try { return client.getObjectMetadata(bucket, key); } + catch (OSSException error) { + if ("NoSuchKey".equals(error.getErrorCode())) return null; + throw new ExternalCoverDownloader.Failure("OSS_HEAD_" + error.getErrorCode()); + } + } + + private void verify(ObjectMetadata metadata, ExternalCoverDownloader.Image image, String sha256) { + if (metadata.getContentLength() != image.bytes().length || !sha256.equals(metadata.getUserMetadata().get("sha256"))) + throw new ExternalCoverDownloader.Failure("OSS_VERIFY_MISMATCH"); + } + + @Override public void close() { client.shutdown(); } +} diff --git a/community-import/src/main/java/com/linkerbot/community/importer/cover/ExternalCoverDownloader.java b/community-import/src/main/java/com/linkerbot/community/importer/cover/ExternalCoverDownloader.java new file mode 100644 index 0000000..b484d18 --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/cover/ExternalCoverDownloader.java @@ -0,0 +1,132 @@ +package com.linkerbot.community.importer.cover; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.InetAddress; +import java.net.URI; +import java.time.Duration; +import java.util.Arrays; + +/** 仅下载公开 HTTPS 图片,不附加任何 GitHub、Gitea 或 OSS 凭据。 */ +public class ExternalCoverDownloader { + public record Image(byte[] bytes, String contentType, String extension) {} + + public static class Failure extends RuntimeException { + private final String code; + + public Failure(String code) { + super(code); + this.code = code; + } + + public String code() { return code; } + } + + private final int maxBytes; + + public ExternalCoverDownloader(int maxBytes) { this.maxBytes = maxBytes; } + + public Image download(String url) throws Exception { + for (int attempt = 1; attempt <= 3; attempt++) { + try { return fetch(url); } + catch (IOException error) { + if (attempt == 3 || !retryable(error) || Thread.currentThread().isInterrupted()) throw error; + Thread.sleep(200L * attempt); + } + } + throw new IllegalStateException("封面下载未完成"); + } + + private boolean retryable(IOException error) { + for (Throwable cause = error; cause != null; cause = cause.getCause()) { + if (cause instanceof java.security.cert.CertificateException + || cause instanceof java.security.cert.CertPathValidatorException + || cause instanceof java.security.cert.CertPathBuilderException) return false; + } + return true; + } + + Image fetch(String url) throws Exception { + URI uri = URI.create(url.trim()); + long deadline = System.nanoTime() + Duration.ofSeconds(45).toNanos(); + for (int redirects = 0; redirects <= 5; redirects++) { + validate(uri); + HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection(); + connection.setInstanceFollowRedirects(false); + connection.setConnectTimeout(10000); + connection.setReadTimeout(15000); + connection.setRequestProperty("User-Agent", "community-cover-migration/1"); + connection.setRequestProperty("Accept", "image/*"); + try { + int status = connection.getResponseCode(); + if (status >= 300 && status < 400) { + String location = connection.getHeaderField("Location"); + if (location == null) throw new Failure("COVER_REDIRECT_INVALID"); + uri = uri.resolve(location); + if (System.nanoTime() > deadline) throw new Failure("COVER_DOWNLOAD_TIMEOUT"); + continue; + } + if (status != 200) throw new Failure("COVER_HTTP_" + status); + long length = connection.getContentLengthLong(); + if (length > maxBytes) throw new Failure("COVER_TOO_LARGE"); + byte[] bytes; + try (InputStream input = connection.getInputStream()) { + bytes = read(input, maxBytes, deadline); + } + if (length >= 0 && length != bytes.length) throw new Failure("COVER_INCOMPLETE"); + return image(bytes); + } finally { + connection.disconnect(); + } + } + throw new Failure("COVER_REDIRECT_LIMIT"); + } + + static void validate(URI uri) throws Exception { + if (!"https".equalsIgnoreCase(uri.getScheme()) || uri.getHost() == null + || uri.getRawUserInfo() != null + || (uri.getPort() != -1 && uri.getPort() != 443)) + throw new Failure("COVER_URL_INVALID"); + for (InetAddress address : InetAddress.getAllByName(uri.getHost())) { + byte[] bytes = address.getAddress(); + boolean reservedIpv4 = bytes.length == 4 && ((bytes[0] & 255) == 0 + || (bytes[0] & 255) >= 224 + || (bytes[0] & 255) == 100 && ((bytes[1] & 255) & 192) == 64 + || (bytes[0] & 255) == 198 && ((bytes[1] & 255) == 18 || (bytes[1] & 255) == 19)); + boolean privateIpv6 = bytes.length == 16 && (bytes[0] & 254) == 252; + if (address.isAnyLocalAddress() || address.isLoopbackAddress() || address.isLinkLocalAddress() + || address.isSiteLocalAddress() || address.isMulticastAddress() || reservedIpv4 || privateIpv6) + throw new Failure("COVER_PRIVATE_ADDRESS"); + } + } + + static byte[] read(InputStream input, int maxBytes, long deadline) throws Exception { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int count; + while ((count = input.read(buffer)) != -1) { + if (System.nanoTime() > deadline || Thread.currentThread().isInterrupted()) + throw new Failure("COVER_DOWNLOAD_TIMEOUT"); + if (count > maxBytes - output.size()) throw new Failure("COVER_TOO_LARGE"); + output.write(buffer, 0, count); + } + return output.toByteArray(); + } + + static Image image(byte[] bytes) { + if (bytes.length >= 3 && (bytes[0] & 255) == 255 && (bytes[1] & 255) == 216 && (bytes[2] & 255) == 255) + return new Image(bytes, "image/jpeg", "jpg"); + if (bytes.length >= 8 && Arrays.equals(Arrays.copyOf(bytes, 8), + new byte[] {(byte) 137, 80, 78, 71, 13, 10, 26, 10})) + return new Image(bytes, "image/png", "png"); + if (bytes.length >= 6 && bytes[0] == 'G' && bytes[1] == 'I' && bytes[2] == 'F' + && bytes[3] == '8' && (bytes[4] == '7' || bytes[4] == '9') && bytes[5] == 'a') + return new Image(bytes, "image/gif", "gif"); + if (bytes.length >= 12 && bytes[0] == 'R' && bytes[1] == 'I' && bytes[2] == 'F' && bytes[3] == 'F' + && bytes[8] == 'W' && bytes[9] == 'E' && bytes[10] == 'B' && bytes[11] == 'P') + return new Image(bytes, "image/webp", "webp"); + throw new Failure("COVER_FORMAT_UNSUPPORTED"); + } +} diff --git a/community-import/src/main/resources/application.yml b/community-import/src/main/resources/application.yml index 9f51282..3a7021d 100644 --- a/community-import/src/main/resources/application.yml +++ b/community-import/src/main/resources/application.yml @@ -17,6 +17,16 @@ mybatis-plus: configuration: map-underscore-to-camel-case: true community: + oss: + # 导入只转换资源地址,永不启用 OSS SDK 或校验上传凭据。 + enabled: false + region: ${OSS_REGION:cn-beijing} + bucket: ${OSS_BUCKET:linkerbot} + access-key-id: ${OSS_ACCESS_KEY_ID:} + access-key-secret: ${OSS_ACCESS_KEY_SECRET:} + public-base-url: ${OSS_PUBLIC_BASE_URL:https://linkerbot.oss-cn-beijing.aliyuncs.com} + legacy-public-base-urls: ${OSS_LEGACY_PUBLIC_BASE_URLS:} + root-directory: vibemake_dev gitea: base-url: ${GITEA_BASE_URL:http://localhost:3000} owner: ${GITEA_ORGANIZATION:community} @@ -26,3 +36,11 @@ logging: root: WARN com.linkerbot.community: INFO com.linkerbot.community.service.mapper: WARN +--- +spring: + config: + activate: + on-profile: prod +community: + oss: + root-directory: vibemake diff --git a/community-import/src/test/java/com/linkerbot/community/importer/ImportCoordinatorTest.java b/community-import/src/test/java/com/linkerbot/community/importer/ImportCoordinatorTest.java index 475818e..698b94e 100644 --- a/community-import/src/test/java/com/linkerbot/community/importer/ImportCoordinatorTest.java +++ b/community-import/src/test/java/com/linkerbot/community/importer/ImportCoordinatorTest.java @@ -11,6 +11,9 @@ import com.linkerbot.community.bean.entity.User; import com.linkerbot.community.common.*; import com.linkerbot.community.service.ProjectService; import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.oss.OssResourcePaths; +import com.linkerbot.community.service.oss.OssObjectStore; +import com.linkerbot.community.service.config.OssProperties; import com.linkerbot.community.service.gitea.*; import com.linkerbot.community.service.mapper.*; @@ -53,6 +56,9 @@ class ImportCoordinatorTest { @Autowired ProjectService service; @Autowired GiteaClient gitea; @Autowired CommunityProperties config; + @Autowired OssResourcePaths resourcePaths; + @Autowired OssProperties oss; + @Autowired org.springframework.context.ApplicationContext context; @Autowired JdbcTemplate jdbc; @TempDir Path directory; private StateStore store; @@ -155,7 +161,7 @@ class ImportCoordinatorTest { private ImportCoordinator worker(ProjectMapper mapper, ProjectService publisher) { return new ImportCoordinator( - mapper, publisher, gitea, config, store, owner, Duration.ofMinutes(30)); + mapper, publisher, gitea, config, resourcePaths, store, owner, Duration.ofMinutes(30)); } @Test @@ -168,6 +174,9 @@ class ImportCoordinatorTest { assertThat(project.getPublishedCommitSha()).isEqualTo(SHA); assertThat(project.getSourceProjectId()).isNull(); assertThat(project.getCoverUrl()).endsWith("a.gif"); + assertThat(oss.isEnabled()).isFalse(); + assertThat(oss.getAccessKeySecret()).isEmpty(); + assertThat(context.getBeansOfType(OssObjectStore.class)).isEmpty(); assertThat(service.view(project, owner.getId()).projectCode()).isEqualTo(entry.projectCode); verify(gitea, never()).createRepository(anyString(), anyBoolean()); verify(gitea).createBranch(remote.repository(), "main", SHA); @@ -175,6 +184,22 @@ class ImportCoordinatorTest { verify(gitea, times(1)).migrate(anyString(), anyString(), any(Duration.class)); } + @Test + void hostedImportCoversNormalizeWithoutOssCredentials() throws Exception { + String path = "/project-covers/111111111111/20261019/a.png"; + Candidate original = entry.candidate; + entry.candidate = new Candidate(original.sheet(), original.row(), original.name(), original.repositoryUrl(), + original.description(), original.projectType(), oss.publicBaseUrl() + "/" + oss.getRootDirectory() + path); + worker(projects, service).run(entry); + assertThat(entry.phase).isEqualTo(Phase.PUBLISHED); + Project project = service.published(entry.projectCode); + assertThat(project.getCoverUrl()).isEqualTo(path); + assertThat(service.view(project, owner.getId()).coverUrl()) + .isEqualTo(oss.publicBaseUrl() + "/" + oss.getRootDirectory() + path); + assertThat(oss.isEnabled()).isFalse(); + assertThat(oss.getAccessKeyId()).isEmpty(); + } + @Test void normalizationErrorResumesOnlyAfterReadOnlyConfirmationOfAppliedState() throws Exception { doThrow(new GiteaFailure(502, "GITEA_ERROR", "写入后响应失败", true)) diff --git a/community-import/src/test/java/com/linkerbot/community/importer/cover/CoverMigratorTest.java b/community-import/src/test/java/com/linkerbot/community/importer/cover/CoverMigratorTest.java new file mode 100644 index 0000000..62268cd --- /dev/null +++ b/community-import/src/test/java/com/linkerbot/community/importer/cover/CoverMigratorTest.java @@ -0,0 +1,134 @@ +package com.linkerbot.community.importer.cover; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkerbot.community.service.config.OssProperties; +import com.linkerbot.community.service.oss.OssResourcePaths; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.InOrder; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; + +public class CoverMigratorTest { + @TempDir Path directory; + private final CoverMigrator.Repository repository = mock(CoverMigrator.Repository.class); + private final CoverMigrator.Downloader downloader = mock(CoverMigrator.Downloader.class); + private final CoverMigrator.Uploader uploader = mock(CoverMigrator.Uploader.class); + private final CoverMigrator.Backup backup = mock(CoverMigrator.Backup.class); + private final OssProperties oss = properties(); + private final OssResourcePaths paths = new OssResourcePaths(oss); + private final ExternalCoverDownloader.Image image = new ExternalCoverDownloader.Image(new byte[] {1, 2, 3}, "image/gif", "gif"); + private static final CoverMigrator.Candidate FIRST = new CoverMigrator.Candidate("1".repeat(12), "https://raw.example.com/a.gif"); + + private static OssProperties properties() { + OssProperties properties = new OssProperties(); + properties.setRootDirectory("vibemake_dev"); + properties.setPublicBaseUrl("https://cdn.example.com"); + return properties; + } + + @Test + void previewSkipsManagedReferencesWithoutAnyDownloadOrWrite() { + when(repository.list()).thenReturn(List.of( + new CoverMigrator.Candidate("1".repeat(12), "/project-covers/a.png"), + new CoverMigrator.Candidate("2".repeat(12), "https://cdn.example.com/vibemake_dev/project-covers/a.png"), FIRST)); + CoverMigrator migrator = new CoverMigrator(repository, paths, downloader, uploader); + assertThat(migrator.candidates(10)).containsExactly(FIRST); + verify(repository, never()).replace(any(), anyString()); + verifyNoInteractions(downloader, uploader); + } + + @Test + void duplicatesDownloadAndUploadOnceAndBackupPrecedesConditionalUpdate() throws Exception { + CoverMigrator.Candidate second = new CoverMigrator.Candidate("2".repeat(12), FIRST.originalUrl()); + when(downloader.download(FIRST.originalUrl())).thenReturn(image); + when(repository.replace(any(), anyString())).thenReturn(true); + List results = new CoverMigrator(repository, paths, downloader, uploader) + .migrate(List.of(FIRST, second), 2, backup); + assertThat(results).allSatisfy(result -> { + assertThat(result.status()).isEqualTo("UPDATED"); + assertThat(result.coverPath()).matches("/project-covers/migrated/[a-f0-9]{64}[.]gif"); + }); + verify(downloader).download(FIRST.originalUrl()); + verify(uploader).upload(startsWith("vibemake_dev/project-covers/migrated/"), same(image), anyString()); + InOrder order = inOrder(backup, repository); + order.verify(backup).record(eq(FIRST), anyString()); + order.verify(repository).replace(eq(FIRST), anyString()); + order.verify(backup).record(eq(second), anyString()); + order.verify(repository).replace(eq(second), anyString()); + } + + @Test + void downloadUploadAndBackupFailuresNeverReplaceDatabaseValues() throws Exception { + when(downloader.download(anyString())).thenThrow(new ExternalCoverDownloader.Failure("COVER_HTTP_404")); + CoverMigrator migrator = new CoverMigrator(repository, paths, downloader, uploader); + assertThat(migrator.migrate(List.of(FIRST), 1, backup).getFirst().code()).isEqualTo("DOWNLOAD:COVER_HTTP_404"); + verifyNoInteractions(uploader, backup); + doReturn(image).when(downloader).download(anyString()); + doThrow(new IllegalStateException()).when(uploader).upload(anyString(), any(), anyString()); + assertThat(migrator.migrate(List.of(FIRST), 1, backup).getFirst().code()).startsWith("UPLOAD:"); + verifyNoInteractions(backup); + doNothing().when(uploader).upload(anyString(), any(), anyString()); + doThrow(new IllegalStateException()).when(backup).record(any(), anyString()); + assertThat(migrator.migrate(List.of(FIRST), 1, backup).getFirst().code()).startsWith("BACKUP:"); + verify(repository, never()).replace(any(), anyString()); + } + + @Test + void conditionalUpdateProtectsCaseOnlyEditsAndIsIdempotent() throws Exception { + JdbcTemplate jdbc = new JdbcTemplate(new DriverManagerDataSource("jdbc:h2:mem:cover_migrate;MODE=MySQL;DB_CLOSE_DELAY=-1", "sa", "")); + jdbc.execute("DROP ALL OBJECTS"); + jdbc.execute("CREATE ALIAS HEX FOR 'com.linkerbot.community.importer.cover.CoverMigratorTest.hex'"); + jdbc.execute("CREATE TABLE project (project_code VARCHAR(32) PRIMARY KEY, cover_url VARCHAR(2048))"); + jdbc.update("INSERT INTO project VALUES (?, ?)", FIRST.projectCode(), FIRST.originalUrl()); + CoverMigrator.JdbcRepository database = new CoverMigrator.JdbcRepository(jdbc); + when(downloader.download(anyString())).thenReturn(image); + CoverMigrator migrator = new CoverMigrator(database, paths, downloader, uploader); + List candidates = migrator.candidates(10); + jdbc.update("UPDATE project SET cover_url = ?", "https://raw.example.com/A.gif"); + assertThat(migrator.migrate(candidates, 1, backup).getFirst().status()).isEqualTo("CHANGED"); + assertThat(jdbc.queryForObject("SELECT cover_url FROM project", String.class)).endsWith("A.gif"); + assertThat(migrator.migrate(migrator.candidates(10), 1, backup).getFirst().status()).isEqualTo("UPDATED"); + assertThat(migrator.candidates(10)).isEmpty(); + } + + @Test + void durableBackupPreservesOriginalValueBeforeWriteAndOptionsRejectConflicts() throws Exception { + Path path = directory.resolve("backup.jsonl"); + try (CoverMigrationApplication.DurableBackup backupFile = new CoverMigrationApplication.DurableBackup(path, new ObjectMapper())) { + backupFile.record(FIRST, "/project-covers/migrated/a.gif"); + assertThat(new ObjectMapper().readTree(Files.readString(path)).path("originalCoverUrl").asText()) + .isEqualTo(FIRST.originalUrl()); + } + assertThat(CoverMigrationApplication.Options.parse(new String[0]).execute()).isFalse(); + assertThatThrownBy(() -> CoverMigrationApplication.Options.parse(new String[] {"--dry-run", "--execute"})) + .isInstanceOf(IllegalArgumentException.class); + } + + public static String hex(String value) { + return value == null ? null : HexFormat.of().formatHex(value.getBytes(StandardCharsets.UTF_8)); + } + + @Test + void explicitRecoverySourcesDoNotReplaceOriginalValuesUsedForBackupAndCompare() { + String alternate = "https://raw.example.com/commit/a.gif"; + assertThat(CoverMigrationApplication.downloadSources(List.of(FIRST), Map.of(FIRST.projectCode(), alternate))) + .containsEntry(FIRST.originalUrl(), alternate); + assertThat(FIRST.originalUrl()).isEqualTo("https://raw.example.com/a.gif"); + assertThat(CoverMigrationApplication.downloadSources(List.of(), Map.of(FIRST.projectCode(), alternate))).isEmpty(); + CoverMigrator.Candidate second = new CoverMigrator.Candidate("2".repeat(12), FIRST.originalUrl()); + assertThatThrownBy(() -> CoverMigrationApplication.downloadSources(List.of(FIRST, second), + Map.of(FIRST.projectCode(), alternate, second.projectCode(), "https://raw.example.com/other.gif"))) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/community-import/src/test/java/com/linkerbot/community/importer/cover/ExternalCoverDownloaderTest.java b/community-import/src/test/java/com/linkerbot/community/importer/cover/ExternalCoverDownloaderTest.java new file mode 100644 index 0000000..7e3beda --- /dev/null +++ b/community-import/src/test/java/com/linkerbot/community/importer/cover/ExternalCoverDownloaderTest.java @@ -0,0 +1,61 @@ +package com.linkerbot.community.importer.cover; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.time.Duration; +import org.junit.jupiter.api.Test; + +class ExternalCoverDownloaderTest { + @Test + void detectsActualImageTypesAndRejectsHtmlDisguisedAsImage() { + assertThat(ExternalCoverDownloader.image(new byte[] {(byte) 255, (byte) 216, (byte) 255}).contentType()) + .isEqualTo("image/jpeg"); + assertThat(ExternalCoverDownloader.image(new byte[] {(byte) 137, 80, 78, 71, 13, 10, 26, 10}).extension()) + .isEqualTo("png"); + assertThat(ExternalCoverDownloader.image("GIF89a-data".getBytes()).extension()).isEqualTo("gif"); + assertThat(ExternalCoverDownloader.image("RIFF1234WEBP-data".getBytes()).extension()).isEqualTo("webp"); + assertThatThrownBy(() -> ExternalCoverDownloader.image("denied".getBytes())) + .isInstanceOfSatisfying(ExternalCoverDownloader.Failure.class, + error -> assertThat(error.code()).isEqualTo("COVER_FORMAT_UNSUPPORTED")); + } + + @Test + void boundedReadRejectsOverflowAndExpiredDownloads() throws Exception { + long deadline = System.nanoTime() + Duration.ofSeconds(5).toNanos(); + assertThat(ExternalCoverDownloader.read(new ByteArrayInputStream(new byte[10]), 10, deadline)).hasSize(10); + assertThatThrownBy(() -> ExternalCoverDownloader.read(new ByteArrayInputStream(new byte[11]), 10, deadline)) + .isInstanceOfSatisfying(ExternalCoverDownloader.Failure.class, + error -> assertThat(error.code()).isEqualTo("COVER_TOO_LARGE")); + assertThatThrownBy(() -> ExternalCoverDownloader.read(new ByteArrayInputStream(new byte[1]), 10, 0)) + .isInstanceOf(ExternalCoverDownloader.Failure.class); + } + + @Test + void rejectsPrivateNetworksCredentialsAndUnsafeRedirectTargets() { + for (String value : new String[] {"https://127.0.0.1/a", "https://10.0.0.1/a", "https://169.254.169.254/a", + "https://[::1]/a", "https://[fd00::1]/a", "https://100.64.0.1/a", "https://198.18.0.1/a", + "http://example.com/a", "https://user:pass@example.com/a", "https://example.com:8443/a"}) + assertThatThrownBy(() -> ExternalCoverDownloader.validate(URI.create(value))) + .isInstanceOf(ExternalCoverDownloader.Failure.class); + } + + @Test + void retriesTransientHandshakesButKeepsCertificateVerificationFailures() throws Exception { + ExternalCoverDownloader downloader = spy(new ExternalCoverDownloader(1024)); + String url = "https://raw.example.com/a.png"; + ExternalCoverDownloader.Image image = new ExternalCoverDownloader.Image(new byte[1], "image/png", "png"); + doThrow(new javax.net.ssl.SSLHandshakeException("transient")) + .doReturn(image).when(downloader).fetch(url); + assertThat(downloader.download(url)).isSameAs(image); + verify(downloader, times(2)).fetch(url); + clearInvocations(downloader); + javax.net.ssl.SSLHandshakeException invalidCertificate = new javax.net.ssl.SSLHandshakeException("untrusted"); + invalidCertificate.initCause(new java.security.cert.CertificateException()); + doThrow(invalidCertificate).when(downloader).fetch(url); + assertThatThrownBy(() -> downloader.download(url)).isSameAs(invalidCertificate); + verify(downloader).fetch(url); + } +} diff --git a/community-service/src/main/java/com/linkerbot/community/service/ai/AiProjectAssistant.java b/community-service/src/main/java/com/linkerbot/community/service/ai/AiProjectAssistant.java index 2f1c2e7..7c933ad 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/ai/AiProjectAssistant.java +++ b/community-service/src/main/java/com/linkerbot/community/service/ai/AiProjectAssistant.java @@ -148,7 +148,7 @@ public class AiProjectAssistant { Project source = projects.published(sourceCode); String name = intent.name() == null ? source.getName() : intent.name(); // 来源的名称可能被其他调用方更新,执行前仍需按现有输入约束校验。 - ProjectInputs.validate(new ProjectInput(name, source.getDescription(), source.getProjectType(), source.getCoverUrl()), properties); + ProjectInputs.validateMetadata(new ProjectInput(name, source.getDescription(), source.getProjectType(), null), properties); ForkInput input = new ForkInput(source.getPublishedCommitSha(), name); progress.accept(new Progress("forking", "正在 Fork 项目到你的工作区")); checkCancelled(); diff --git a/community-service/src/main/java/com/linkerbot/community/service/config/OssProperties.java b/community-service/src/main/java/com/linkerbot/community/service/config/OssProperties.java index 8092f07..1db00c9 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/config/OssProperties.java +++ b/community-service/src/main/java/com/linkerbot/community/service/config/OssProperties.java @@ -2,6 +2,8 @@ package com.linkerbot.community.service.config; import jakarta.validation.constraints.AssertTrue; import java.net.URI; +import java.util.ArrayList; +import java.util.List; import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -22,6 +24,13 @@ public class OssProperties { private String accessKeySecret = ""; private String publicBaseUrl = ""; private String rootDirectory = ""; + private List legacyPublicBaseUrls = new ArrayList<>(); + + @AssertTrue(message = "历史 OSS/CDN 基础地址必须是无凭据、查询参数及片段的 HTTPS 地址") + public boolean isLegacyPublicBaseUrlsValid() { + return legacyPublicBaseUrls != null + && legacyPublicBaseUrls.stream().allMatch(value -> validBaseUrl(value, true)); + } @AssertTrue(message = "开启 OSS 时必须配置合法的地域 ID(如 cn-beijing)、Bucket、凭据及根目录;Endpoint 为可选 HTTPS 地址") public boolean isConfigurationValid() { diff --git a/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java b/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java index 3922a6a..d7aec3b 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java +++ b/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java @@ -19,6 +19,7 @@ import com.linkerbot.community.service.fork.ForkExecutor; import com.linkerbot.community.service.fork.GitSnapshotCopier; import com.linkerbot.community.service.mapper.*; import com.linkerbot.community.service.oss.UrdfResources; +import com.linkerbot.community.service.oss.OssResourcePaths; import com.linkerbot.community.service.support.ProjectLocks; import com.linkerbot.community.service.support.ProjectInputs; import org.springframework.dao.DuplicateKeyException; @@ -44,7 +45,8 @@ public class ProjectServiceImpl implements ProjectService { private final GiteaClient gitea; private final CommunityProperties config; private final ProjectLocks locks; - // 离线导入只加载仓库和数据库组件,没有 URDF 时无需初始化 OSS 模块。 + private final OssResourcePaths resourcePaths; + // 离线导入只需地址转换配置,没有 URDF 时无需初始化 OSS 客户端。 private final ObjectProvider urdf; private final ObjectProvider forks; private final ObjectProvider snapshots; @@ -61,7 +63,7 @@ public class ProjectServiceImpl implements ProjectService { } private String normalizeCoverUrl(String value) { - return ProjectInputs.coverUrl(value); + return resourcePaths.coverStorage(value); } @Override @@ -130,13 +132,24 @@ public class ProjectServiceImpl implements ProjectService { .eq(ProjectLike::getProjectId, project.getId()) .eq(ProjectLike::getUserId, currentUser)) > 0; + String coverUrl; + String urdfMainUrl; + String urdfBaseUrl; + try { + coverUrl = resourcePaths.coverUrl(project.getCoverUrl()); + urdfMainUrl = urdfUrl(project.getUrdfMainObjectKey()); + urdfBaseUrl = urdfUrl(project.getUrdfPrefix()); + } catch (BusinessException error) { + log.warn("项目资源地址生成失败 projectCode={} code={}", project.getProjectCode(), error.code()); + throw error; + } return new ProjectView( project.getProjectCode(), project.getName(), project.getDescription(), - project.getCoverUrl(), - urdfUrl(project.getUrdfMainObjectKey()), - urdfUrl(project.getUrdfPrefix()), + coverUrl, + urdfMainUrl, + urdfBaseUrl, project.getProjectType(), users.publicView(project.getOwnerUserId()), project.getRepoStatus(), @@ -196,13 +209,15 @@ public class ProjectServiceImpl implements ProjectService { } private Project initialize(long userId, ProjectInput input, Project source, ForkDeadline deadline) { - validateInput(input); + if (source == null) validateInput(input); + else ProjectInputs.validateMetadata(input, config); Project project = new Project(); project.setProjectCode(ProjectCodes.next()); project.setOwnerUserId(userId); project.setName(input.name().trim()); project.setDescription(input.description() == null ? "" : input.description()); - project.setCoverUrl(normalizeCoverUrl(input.coverUrl())); + project.setCoverUrl(source == null ? normalizeCoverUrl(input.coverUrl()) + : resourcePaths.coverReference(source.getCoverUrl())); project.setProjectType(input.projectType()); project.setGiteaOwner(config.getGitea().getOwner()); project.setGiteaRepoName("p_" + project.getProjectCode()); @@ -258,7 +273,7 @@ public class ProjectServiceImpl implements ProjectService { ProjectInput data = new ProjectInput( input.name(), source.getDescription(), source.getProjectType(), source.getCoverUrl()); - validateInput(data); + ProjectInputs.validateMetadata(data, config); ForkExecutor executor = forks.getIfAvailable(); if (executor == null) throw new BusinessException(503, "FORK_UNAVAILABLE", "当前进程未开启 Fork"); return executor.execute(deadline -> { diff --git a/community-service/src/main/java/com/linkerbot/community/service/oss/OssPostSigner.java b/community-service/src/main/java/com/linkerbot/community/service/oss/OssPostSigner.java index 2f7ac79..a90f323 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/oss/OssPostSigner.java +++ b/community-service/src/main/java/com/linkerbot/community/service/oss/OssPostSigner.java @@ -33,6 +33,7 @@ public class OssPostSigner { private final OssProperties properties; private final ObjectMapper json; private final Clock clock; + private final OssResourcePaths resourcePaths; @Autowired public OssPostSigner(OssProperties properties, ObjectMapper json) { @@ -43,6 +44,7 @@ public class OssPostSigner { this.properties = properties; this.json = json; this.clock = clock; + this.resourcePaths = new OssResourcePaths(properties); } public CoverUploadSignature sign(String userCode, String contentType, String extension) { @@ -50,7 +52,7 @@ public class OssPostSigner { + DATE.format(clock.instant()) + "/" + Codes.next() + "." + extension; SignedPost post = post(key, contentType, 1, MAX_FILE_BYTES, VALIDITY, false); return new CoverUploadSignature(properties.uploadUrl(), key, - properties.publicBaseUrl() + "/" + key, post.expiresAt(), post.formFields()); + resourcePaths.publicUrl(key), post.expiresAt(), post.formFields()); } public record SignedPost(LocalDateTime expiresAt, Map formFields) {} diff --git a/community-service/src/main/java/com/linkerbot/community/service/oss/OssResourcePaths.java b/community-service/src/main/java/com/linkerbot/community/service/oss/OssResourcePaths.java new file mode 100644 index 0000000..d7af643 --- /dev/null +++ b/community-service/src/main/java/com/linkerbot/community/service/oss/OssResourcePaths.java @@ -0,0 +1,121 @@ +package com.linkerbot.community.service.oss; + +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.common.FilePaths; +import com.linkerbot.community.service.config.OssProperties; +import com.linkerbot.community.service.support.ProjectInputs; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.springframework.web.util.UriUtils; + +/** 仅转换资源地址,不访问 OSS,也不依赖上传凭据。存储路径为未编码的环境相对路径。 */ +@Component +@RequiredArgsConstructor +public class OssResourcePaths { + private final OssProperties oss; + + /** 外部请求仍只接受 HTTPS URL;仅转换明确属于当前环境的托管封面。 */ + public String coverStorage(String value) { + String url = ProjectInputs.coverUrl(value); + if (url == null) return null; + URI candidate = URI.create(url); + if (candidate.getRawQuery() != null || candidate.getRawFragment() != null) return url; + for (String baseValue : managedBases()) { + URI base = URI.create(baseValue); + if (!sameOrigin(candidate, base)) continue; + String prefix = base.getRawPath().replaceAll("/+$", "") + + "/" + oss.getRootDirectory() + "/project-covers/"; + if (!candidate.getRawPath().startsWith(prefix)) continue; + try { + String[] segments = candidate.getRawPath().substring(prefix.length()).split("/", -1); + for (int index = 0; index < segments.length; index++) { + segments[index] = UriUtils.decode(segments[index], StandardCharsets.UTF_8); + // 保留含歧义分隔符或 UTF-8 替换字符的原地址,避免引用到不同对象。 + if (segments[index].contains("/") || segments[index].indexOf('\uFFFD') >= 0) return url; + } + return storagePath("/project-covers/" + String.join("/", segments)); + } catch (IllegalArgumentException | BusinessException error) { + // 非规范地址保留原语义,不进行路径折叠或二次解码。 + return url; + } + } + return url; + } + + /** 内部 Fork 可以复用已持久化的路径,不走客户端 URL 校验。 */ + public String coverReference(String value) { + if (value != null && value.startsWith("/project-covers/")) return storagePath(value); + return coverStorage(value); + } + + public String coverUrl(String value) { + String reference = coverReference(value); + return reference != null && reference.startsWith("/project-covers/") + ? publicUrl(reference) : reference; + } + + /** 兼容当前环境的旧 key;绝不把其他环境的 key 重定位至当前环境。 */ + public String storagePath(String value) { + if (value == null || value.isEmpty()) throw invalidPath(); + String path = value; + String root = oss.getRootDirectory(); + if (root != null && !root.isBlank() && path.startsWith(root + "/")) + path = path.substring(root.length()); + if (!path.startsWith("/project-covers/") && !path.startsWith("/urdf/")) throw invalidPath(); + String relative = path.substring(1); + if (relative.endsWith("/")) relative = relative.substring(0, relative.length() - 1); + try { + FilePaths.file(relative); + } catch (BusinessException error) { + throw invalidPath(); + } + return path; + } + + public String objectKey(String value) { + String path = storagePath(value); + String root = oss.getRootDirectory(); + if (root == null || !root.matches("[A-Za-z0-9_-]+")) + throw new BusinessException(503, "OSS_UNAVAILABLE", "OSS 资源根目录未正确配置"); + return root + path; + } + + public String publicUrl(String value) { + if (value == null || value.isEmpty()) return null; + String[] segments = objectKey(value).split("/", -1); + for (int index = 0; index < segments.length; index++) + segments[index] = UriUtils.encodePathSegment(segments[index], StandardCharsets.UTF_8); + return oss.publicBaseUrl() + "/" + String.join("/", segments); + } + + private List managedBases() { + List bases = new ArrayList<>(oss.getLegacyPublicBaseUrls()); + if (oss.getPublicBaseUrl() != null && !oss.getPublicBaseUrl().isBlank()) + bases.add(oss.getPublicBaseUrl()); + if (oss.getBucket() != null && !oss.getBucket().isBlank() + && oss.getRegion() != null && !oss.getRegion().isBlank()) { + bases.add("https://" + oss.getBucket() + ".oss-" + oss.getRegion() + ".aliyuncs.com"); + bases.add(oss.uploadUrl()); + } + return bases; + } + + private boolean sameOrigin(URI first, URI second) { + return second.getHost() != null + && first.getScheme().equalsIgnoreCase(second.getScheme()) + && first.getHost().equalsIgnoreCase(second.getHost()) + && port(first) == port(second); + } + + private int port(URI uri) { + return uri.getPort() == -1 ? 443 : uri.getPort(); + } + + private BusinessException invalidPath() { + return new BusinessException(500, "RESOURCE_PATH_INVALID", "资源存储路径异常"); + } +} diff --git a/community-service/src/main/java/com/linkerbot/community/service/oss/UrdfResources.java b/community-service/src/main/java/com/linkerbot/community/service/oss/UrdfResources.java index 4291cc2..e504fee 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/oss/UrdfResources.java +++ b/community-service/src/main/java/com/linkerbot/community/service/oss/UrdfResources.java @@ -25,7 +25,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.MediaTypeFactory; import org.springframework.stereotype.Component; -import org.springframework.web.util.UriUtils; @Component @RequiredArgsConstructor @@ -37,6 +36,7 @@ public class UrdfResources { private final OssPostSigner signer; private final OssObjectStore store; private final ObjectMapper json; + private final OssResourcePaths paths; public void requireEnabled() { if (!oss.isEnabled()) @@ -47,7 +47,7 @@ public class UrdfResources { public String validate(String prefix, List files) { if (files == null) throw BusinessException.badRequest("必须提供 URDF 文件清单"); if (files.size() > limits.getMaxFiles()) throw tooLarge(); - Set paths = new HashSet<>(); + Set seenPaths = new HashSet<>(); long total = 0; String main = null; int mainCount = 0; @@ -55,8 +55,8 @@ public class UrdfResources { if (file == null || file.sizeBytes() == null || file.sizeBytes() < 0) throw BusinessException.badRequest("文件大小不合法"); String path = FilePaths.file(file.relativePath()); - if (!paths.add(path)) throw BusinessException.badRequest("上传路径不能重复"); - String key = prefix + path; + if (!seenPaths.add(path)) throw BusinessException.badRequest("上传路径不能重复"); + String key = paths.objectKey(prefix + path); if (key.length() > 512 || key.getBytes(StandardCharsets.UTF_8).length > 1023) throw BusinessException.badRequest("URDF 文件对象键过长"); if (file.sizeBytes() > limits.getMaxFileBytes() @@ -83,7 +83,7 @@ public class UrdfResources { if (projectCode == null || !projectCode.matches("[a-z0-9]{12}|[a-z0-9]{32}") || batchCode == null || !batchCode.matches("[a-f0-9]{32}")) throw BusinessException.badRequest("URDF 项目或批次编码不合法"); - return oss.getRootDirectory() + "/urdf/" + projectCode + "/" + batchCode + "/"; + return "/urdf/" + projectCode + "/" + batchCode + "/"; } public UrdfUploadSignature sign(String projectCode, List files) { @@ -97,7 +97,7 @@ public class UrdfResources { String type = file.relativePath().equals(main) ? "application/xml" : MediaTypeFactory.getMediaType(file.relativePath().toLowerCase(Locale.ROOT)) .map(Object::toString).orElse("application/octet-stream"); - String key = prefix + file.relativePath(); + String key = paths.objectKey(prefix + file.relativePath()); OssPostSigner.SignedPost post = signer.signObject(key, type, file.sizeBytes()); if (expiration == null || post.expiresAt().isBefore(expiration)) expiration = post.expiresAt(); signatures.add(new UrdfFileSignature(file.relativePath(), key, post.formFields())); @@ -116,10 +116,11 @@ public class UrdfResources { private OssObjectStore.Metadata checkedMetadata(String prefix, UrdfFile file) { ForkDeadline.checkpoint(); - OssObjectStore.Metadata metadata = store.head(prefix + file.relativePath()); + String key = paths.objectKey(prefix + file.relativePath()); + OssObjectStore.Metadata metadata = store.head(key); if (metadata.sizeBytes() != file.sizeBytes()) { log.warn("URDF 文件大小不符 objectKey={} expectedBytes={} actualBytes={}", - prefix + file.relativePath(), file.sizeBytes(), metadata.sizeBytes()); + key, file.sizeBytes(), metadata.sizeBytes()); throw new BusinessException(400, "URDF_FILE_SIZE_MISMATCH", "URDF 文件大小不符,请重新上传"); } return metadata; @@ -144,13 +145,22 @@ public class UrdfResources { source.getProjectCode(), error.getClass().getSimpleName()); throw new BusinessException(500, "URDF_MANIFEST_INVALID", "URDF 文件清单异常"); } - String sourcePrefix = source.getUrdfPrefix(); - String root = oss.getRootDirectory() + "/urdf/" + source.getProjectCode() + "/"; + String sourcePrefix; + String sourceMain; + try { + sourcePrefix = paths.storagePath(source.getUrdfPrefix()); + sourceMain = paths.storagePath(source.getUrdfMainObjectKey()); + } catch (BusinessException error) { + log.warn("Fork URDF 路径异常 sourceProjectCode={} projectCode={} code={}", + source.getProjectCode(), target.getProjectCode(), error.code()); + throw new BusinessException(500, "URDF_MANIFEST_INVALID", "URDF 资源路径异常"); + } + String root = "/urdf/" + source.getProjectCode() + "/"; if (sourcePrefix == null || !sourcePrefix.startsWith(root) || !sourcePrefix.substring(root.length()).matches("[a-f0-9]{32}/")) throw new BusinessException(500, "URDF_MANIFEST_INVALID", "URDF 资源目录异常"); String main = validate(sourcePrefix, files); - if (!source.getUrdfMainObjectKey().equals(sourcePrefix + main)) + if (!sourceMain.equals(sourcePrefix + main)) throw new BusinessException(500, "URDF_MANIFEST_INVALID", "URDF 主文件记录异常"); String targetPrefix = prefix(target.getProjectCode(), Codes.next()); validate(targetPrefix, files); @@ -159,7 +169,8 @@ public class UrdfResources { for (UrdfFile file : files) { OssObjectStore.Metadata metadata = checkedMetadata(sourcePrefix, file); ForkDeadline.checkpoint(); - store.copy(sourcePrefix + file.relativePath(), targetPrefix + file.relativePath(), metadata.etag()); + store.copy(paths.objectKey(sourcePrefix + file.relativePath()), + paths.objectKey(targetPrefix + file.relativePath()), metadata.etag()); } ForkDeadline.checkpoint(); target.setUrdfPrefix(targetPrefix); @@ -170,11 +181,7 @@ public class UrdfResources { } public String url(String objectKey) { - if (objectKey == null || objectKey.isEmpty()) return null; - String[] segments = objectKey.split("/", -1); - for (int index = 0; index < segments.length; index++) - segments[index] = UriUtils.encodePathSegment(segments[index], StandardCharsets.UTF_8); - return oss.publicBaseUrl() + "/" + String.join("/", segments); + return paths.publicUrl(objectKey); } private long totalBytes(List files) { diff --git a/community-service/src/main/java/com/linkerbot/community/service/support/ProjectInputs.java b/community-service/src/main/java/com/linkerbot/community/service/support/ProjectInputs.java index bd43cac..5a481a8 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/support/ProjectInputs.java +++ b/community-service/src/main/java/com/linkerbot/community/service/support/ProjectInputs.java @@ -10,13 +10,17 @@ public final class ProjectInputs { private ProjectInputs() {} public static void validate(ProjectInput input, CommunityProperties config) { + validateMetadata(input, config); + coverUrl(input.coverUrl()); + } + + public static void validateMetadata(ProjectInput input, CommunityProperties config) { if (input.name() == null || input.name().isBlank() || input.name().length() > 100) throw BusinessException.badRequest("项目名称必须为 1~100 字符"); if (input.description() != null && input.description().length() > 2000) throw BusinessException.badRequest("项目描述不能超过 2000 字符"); if (!config.getTypes().containsKey(input.projectType())) throw BusinessException.badRequest("项目类型不在可选列表中"); - coverUrl(input.coverUrl()); } public static String coverUrl(String value) { diff --git a/community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java b/community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java index 2fd9ce0..5310029 100644 --- a/community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java +++ b/community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java @@ -16,6 +16,8 @@ import com.linkerbot.community.service.fork.GitSnapshotCopier; import com.linkerbot.community.service.impl.ProjectServiceImpl; import com.linkerbot.community.service.mapper.*; import com.linkerbot.community.service.oss.UrdfResources; +import com.linkerbot.community.service.oss.OssResourcePaths; +import com.linkerbot.community.service.config.OssProperties; import com.linkerbot.community.service.support.ProjectLocks; import org.junit.jupiter.api.Test; @@ -89,6 +91,7 @@ class ProjectCodeAllocationTest { gitea, new CommunityProperties(), new ProjectLocks(), + new OssResourcePaths(new OssProperties()), new StaticListableBeanFactory().getBeanProvider(UrdfResources.class), new StaticListableBeanFactory().getBeanProvider(ForkExecutor.class), new StaticListableBeanFactory().getBeanProvider(GitSnapshotCopier.class)); diff --git a/community-service/src/test/java/com/linkerbot/community/service/fork/ProjectForkStateTest.java b/community-service/src/test/java/com/linkerbot/community/service/fork/ProjectForkStateTest.java index cabac5a..351d6a4 100644 --- a/community-service/src/test/java/com/linkerbot/community/service/fork/ProjectForkStateTest.java +++ b/community-service/src/test/java/com/linkerbot/community/service/fork/ProjectForkStateTest.java @@ -15,6 +15,8 @@ import com.linkerbot.community.service.gitea.GiteaClient; import com.linkerbot.community.service.impl.ProjectServiceImpl; import com.linkerbot.community.service.mapper.*; import com.linkerbot.community.service.oss.UrdfResources; +import com.linkerbot.community.service.oss.OssResourcePaths; +import com.linkerbot.community.service.config.OssProperties; import com.linkerbot.community.service.support.ProjectLocks; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -73,7 +75,7 @@ class ProjectForkStateTest { factory.addBean("snapshots", copier); ProjectLocks locks = new ProjectLocks(); ProjectServiceImpl service = new ProjectServiceImpl(projects, mock(LikeMapper.class), mock(CommentMapper.class), users, - gitea, new CommunityProperties(), locks, factory.getBeanProvider(UrdfResources.class), + gitea, new CommunityProperties(), locks, new OssResourcePaths(new OssProperties()), factory.getBeanProvider(UrdfResources.class), factory.getBeanProvider(ForkExecutor.class), factory.getBeanProvider(GitSnapshotCopier.class)); try { assertThatThrownBy(() -> service.fork(1, source.getProjectCode(), new ForkInput(source.getPublishedCommitSha(), "副本"))) diff --git a/community-service/src/test/java/com/linkerbot/community/service/oss/OssPropertiesTest.java b/community-service/src/test/java/com/linkerbot/community/service/oss/OssPropertiesTest.java index 025ee4c..fe69787 100644 --- a/community-service/src/test/java/com/linkerbot/community/service/oss/OssPropertiesTest.java +++ b/community-service/src/test/java/com/linkerbot/community/service/oss/OssPropertiesTest.java @@ -74,4 +74,21 @@ class OssPropertiesTest { enabled().withPropertyValues("community.oss." + property) .run(context -> assertThat(context).hasFailed()); } + + @Test + void bindsLegacyBasesAndValidatesThemEvenWhenUploadsAreDisabled() { + runner.withPropertyValues("community.oss.legacy-public-base-urls=") + .run(context -> { + assertThat(context).hasNotFailed(); + assertThat(context.getBean(OssProperties.class).getLegacyPublicBaseUrls()).isEmpty(); + }); + runner.withPropertyValues("community.oss.legacy-public-base-urls=https://old.example.com/assets,https://old-bucket.example.com") + .run(context -> { + assertThat(context).hasNotFailed(); + assertThat(context.getBean(OssProperties.class).getLegacyPublicBaseUrls()).hasSize(2); + }); + for (String invalid : new String[] {"http://old.example.com", "https://old.example.com?token=example", "https://user:pass@old.example.com"}) + runner.withPropertyValues("community.oss.legacy-public-base-urls=" + invalid) + .run(context -> assertThat(context).hasFailed()); + } } diff --git a/community-service/src/test/java/com/linkerbot/community/service/oss/OssResourcePathsTest.java b/community-service/src/test/java/com/linkerbot/community/service/oss/OssResourcePathsTest.java new file mode 100644 index 0000000..73e2937 --- /dev/null +++ b/community-service/src/test/java/com/linkerbot/community/service/oss/OssResourcePathsTest.java @@ -0,0 +1,89 @@ +package com.linkerbot.community.service.oss; + +import static org.assertj.core.api.Assertions.*; + +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.config.OssProperties; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class OssResourcePathsTest { + private final OssProperties oss = new OssProperties(); + private final OssResourcePaths paths = new OssResourcePaths(oss); + private static final String COVER = "/project-covers/111111111111/20261019/a.png"; + + @BeforeEach + void configure() { + oss.setRootDirectory("vibemake_dev"); + oss.setRegion("cn-beijing"); + oss.setBucket("examplebucket"); + oss.setPublicBaseUrl("https://cdn.example.com/assets/"); + oss.setLegacyPublicBaseUrls(List.of("https://old.example.com/previous/")); + } + + @ParameterizedTest + @ValueSource(strings = {"https://cdn.example.com/assets", "https://CDN.EXAMPLE.COM:443/assets", + "https://examplebucket.oss-cn-beijing.aliyuncs.com", "https://old.example.com/previous"}) + void normalizesOnlyManagedCoversAndRendersWithCurrentCdn(String base) { + String original = base + "/vibemake_dev" + COVER; + assertThat(paths.coverStorage(" " + original + " ")).isEqualTo(COVER); + assertThat(paths.coverUrl(original)).isEqualTo("https://cdn.example.com/assets/vibemake_dev" + COVER); + assertThat(paths.coverReference(COVER)).isEqualTo(COVER); + assertThat(paths.coverUrl(COVER)).isEqualTo(paths.coverUrl(original)); + } + + @ParameterizedTest + @ValueSource(strings = {"https://external.example.com/vibemake_dev", "https://cdn.example.com/assets/vibemake", + "https://cdn.example.com.evil/assets/vibemake_dev", "https://cdn.example.com:444/assets/vibemake_dev", + "https://cdn.example.com/assets-other/vibemake_dev", "https://cdn.example.com/assets/vibemake_dev-extra"}) + void keepsExternalOriginsOtherEnvironmentsAndWrongPathBoundaries(String base) { + String url = base + COVER; + assertThat(paths.coverStorage(url)).isEqualTo(url); + assertThat(paths.coverUrl(url)).isEqualTo(url); + } + + @ParameterizedTest + @ValueSource(strings = {"?version=2", "#preview", "?", "#"}) + void preservesQueryAndFragmentIncludingEmptyValues(String suffix) { + String url = "https://cdn.example.com/assets/vibemake_dev" + COVER + suffix; + assertThat(paths.coverStorage(url)).isEqualTo(url); + assertThat(paths.coverUrl(url)).isEqualTo(url); + } + + @Test + void encodesOnlyOncePreservesPlusAndNeverDropsCdnBasePath() { + String stored = "/project-covers/user/日期/a #?%+.png"; + String url = "https://cdn.example.com/assets/vibemake_dev/project-covers/user/" + + "%E6%97%A5%E6%9C%9F/a%20%23%3F%25+.png"; + assertThat(paths.coverUrl(stored)).isEqualTo(url); + assertThat(paths.coverStorage(url)).isEqualTo(stored); + assertThat(paths.coverStorage("https://cdn.example.com/assets/vibemake_dev/project-covers/a%252Fb.png")) + .isEqualTo("/project-covers/a%2Fb.png"); + } + + @ParameterizedTest + @ValueSource(strings = {"../a.png", "%2e%2e/a.png", "user%2Fa.png", "user//a.png", "user/%5Ca.png", "user/%FF.png"}) + void doesNotRewriteAmbiguousOrUnsafeUrlPaths(String suffix) { + String url = "https://cdn.example.com/assets/vibemake_dev/project-covers/" + suffix; + assertThat(paths.coverStorage(url)).isEqualTo(url); + } + + @Test + void supportsLegacyUrdfKeysAndDirectoryUrlsWithoutReinterpretingOtherEnvironments() { + String relative = "/urdf/7m4k9x2p8q6r/" + "a".repeat(32) + "/"; + assertThat(paths.storagePath("vibemake_dev" + relative)).isEqualTo(relative); + assertThat(paths.objectKey(relative)).isEqualTo("vibemake_dev" + relative); + assertThat(paths.publicUrl(relative)).isEqualTo(paths.publicUrl("vibemake_dev" + relative)).endsWith("/"); + assertThatThrownBy(() -> paths.publicUrl("vibemake" + relative)).isInstanceOf(BusinessException.class); + oss.setRootDirectory("vibemake"); + assertThat(paths.publicUrl(relative)).isEqualTo("https://cdn.example.com/assets/vibemake" + relative); + assertThatThrownBy(() -> paths.publicUrl("vibemake_dev" + relative)).isInstanceOf(BusinessException.class); + assertThat(paths.publicUrl("")).isNull(); + assertThat(paths.coverStorage(null)).isNull(); + assertThatThrownBy(() -> paths.coverStorage(COVER)).isInstanceOf(BusinessException.class); + assertThatThrownBy(() -> paths.objectKey("/urdf/../other/a.urdf")).isInstanceOf(BusinessException.class); + } +} diff --git a/community-service/src/test/java/com/linkerbot/community/service/oss/UrdfResourcesTest.java b/community-service/src/test/java/com/linkerbot/community/service/oss/UrdfResourcesTest.java index 30f6cbe..bdc613e 100644 --- a/community-service/src/test/java/com/linkerbot/community/service/oss/UrdfResourcesTest.java +++ b/community-service/src/test/java/com/linkerbot/community/service/oss/UrdfResourcesTest.java @@ -11,13 +11,15 @@ import com.linkerbot.community.service.config.OssProperties; import com.linkerbot.community.service.config.UrdfProperties; import java.util.List; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; class UrdfResourcesTest { private final OssProperties oss = new OssProperties(); private final UrdfProperties limits = new UrdfProperties(); private final OssObjectStore store = mock(OssObjectStore.class); private final UrdfResources resources = new UrdfResources( - oss, limits, mock(OssPostSigner.class), store, new ObjectMapper()); + oss, limits, mock(OssPostSigner.class), store, new ObjectMapper(), new OssResourcePaths(oss)); @Test void acceptsExactLimitsAndRejectsOverflowCountsAndCrossProjectBatches() { @@ -39,6 +41,7 @@ class UrdfResourcesTest { @Test void encodesUrlSegmentsWithoutEncodingSlashesAndKeepsEmptyUrlsNull() { oss.setPublicBaseUrl("https://cdn.example.com/assets/"); + oss.setRootDirectory("vibemake"); assertThat(resources.url("vibemake/urdf/目录/a #?%+.urdf")) .isEqualTo("https://cdn.example.com/assets/vibemake/urdf/%E7%9B%AE%E5%BD%95/a%20%23%3F%25+.urdf"); assertThat(resources.url("vibemake/urdf/a/")).endsWith("/a/"); @@ -46,28 +49,63 @@ class UrdfResourcesTest { assertThat(resources.url(null)).isNull(); } - @Test - void failedCopyLeavesTargetEmptyAndSuccessfulCopyUsesSourceEtag() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void failedCopyLeavesTargetEmptyAndSuccessfulCopyUsesSourceEtag(boolean legacy) { oss.setEnabled(true); oss.setRootDirectory("vibemake_dev"); Project source = new Project(); source.setProjectCode("7m4k9x2p8q6r"); - String sourcePrefix = resources.prefix(source.getProjectCode(), "a".repeat(32)); + String relativePrefix = resources.prefix(source.getProjectCode(), "a".repeat(32)); + String sourcePrefix = legacy ? "vibemake_dev" + relativePrefix : relativePrefix; + String sourceKeyPrefix = "vibemake_dev" + relativePrefix; source.setUrdfPrefix(sourcePrefix); source.setUrdfMainObjectKey(sourcePrefix + "a.urdf"); source.setUrdfFiles(resources.manifest(List.of(new UrdfFile("a.urdf", 12L)))); Project target = new Project(); target.setProjectCode("222222222222"); - when(store.head(sourcePrefix + "a.urdf")).thenReturn(new OssObjectStore.Metadata(12, "source-etag")); + when(store.head(sourceKeyPrefix + "a.urdf")).thenReturn(new OssObjectStore.Metadata(12, "source-etag")); doThrow(new BusinessException(502, "OSS_ERROR", "copy failed")).when(store).copy(anyString(), anyString(), anyString()); assertThatThrownBy(() -> resources.copy(source, target)).isInstanceOf(BusinessException.class); assertThat(target.getUrdfPrefix()).isEmpty(); assertThat(target.getUrdfMainObjectKey()).isEmpty(); reset(store); - when(store.head(sourcePrefix + "a.urdf")).thenReturn(new OssObjectStore.Metadata(12, "source-etag")); + when(store.head(sourceKeyPrefix + "a.urdf")).thenReturn(new OssObjectStore.Metadata(12, "source-etag")); resources.copy(source, target); - verify(store).copy(sourcePrefix + "a.urdf", target.getUrdfPrefix() + "a.urdf", "source-etag"); - assertThat(target.getUrdfPrefix()).startsWith("vibemake_dev/urdf/222222222222/"); + verify(store).copy(sourceKeyPrefix + "a.urdf", "vibemake_dev" + target.getUrdfPrefix() + "a.urdf", "source-etag"); + assertThat(target.getUrdfPrefix()).startsWith("/urdf/222222222222/"); assertThat(target.getUrdfMainObjectKey()).isEqualTo(target.getUrdfPrefix() + "a.urdf"); } + + @Test + void rejectsForeignEnvironmentBeforeAnyOssRequest() { + oss.setEnabled(true); + oss.setRootDirectory("vibemake_dev"); + Project source = new Project(); + source.setProjectCode("7m4k9x2p8q6r"); + source.setUrdfPrefix("vibemake/urdf/7m4k9x2p8q6r/" + "a".repeat(32) + "/"); + source.setUrdfMainObjectKey(source.getUrdfPrefix() + "a.urdf"); + source.setUrdfFiles(resources.manifest(List.of(new UrdfFile("a.urdf", 12L)))); + Project target = new Project(); + target.setProjectCode("222222222222"); + assertThatThrownBy(() -> resources.copy(source, target)) + .isInstanceOfSatisfying(BusinessException.class, + error -> assertThat(error.code()).isEqualTo("URDF_MANIFEST_INVALID")); + verifyNoInteractions(store); + } + + @Test + void lengthLimitIncludesEnvironmentRootAndUsesUtf8Bytes() { + oss.setRootDirectory("vibemake_dev"); + String prefix = resources.prefix("7m4k9x2p8q6r", "a".repeat(32)); + int remaining = 512 - ("vibemake_dev" + prefix).length() - ".urdf".length(); + assertThat(resources.validate(prefix, List.of(new UrdfFile("a".repeat(remaining) + ".urdf", 1L)))) + .endsWith(".urdf"); + assertThatThrownBy(() -> resources.validate(prefix, + List.of(new UrdfFile("a".repeat(remaining + 1) + ".urdf", 1L)))) + .isInstanceOf(BusinessException.class); + assertThatThrownBy(() -> resources.validate(prefix, + List.of(new UrdfFile("中".repeat(340) + ".urdf", 1L)))) + .isInstanceOf(BusinessException.class); + } } diff --git a/docs/community-api.md b/docs/community-api.md index 54699ad..16f9b62 100644 --- a/docs/community-api.md +++ b/docs/community-api.md @@ -568,7 +568,9 @@ async function replaceProjectCover(project: Project, file: File, token: string) 服务端启用 `OSS_ENABLED=true`,配置 `OSS_REGION`、`OSS_BUCKET`、`OSS_ACCESS_KEY_ID`、`OSS_ACCESS_KEY_SECRET`;地域 ID 使用 `cn-beijing` 等编码,不带 `oss-` 前缀。Endpoint 自动生成为 `https://oss-{region}.aliyuncs.com`,例如北京为 `https://oss-cn-beijing.aliyuncs.com`,无需配置 Endpoint 环境变量。特殊域名场景可显式设置 `community.oss.endpoint`,优先使用配置值,要求为公网 HTTPS 地址,不带 Bucket、路径、凭据或查询参数。`OSS_PUBLIC_BASE_URL` 可选。启用时配置缺失或不合法会停止启动。密钥不提交仓库、不返回浏览器;浏览器只收到 AccessKeyId 派生的 credential、Policy 和签名。 -公开基础地址可以是 OSS 或 CDN 域名,可带基础路径;最终地址为 `{公开基础地址}/{objectKey}`,CDN 路径需正确映射到同一对象。留空时为 `https://{Bucket}.{Endpoint 主机名}/{objectKey}`。数据库保存固定地址,不保存临时签名下载链接。 +公开基础地址可以是 OSS 或 CDN 域名,可带基础路径;最终地址为 `{公开基础地址}/{objectKey}`,CDN 路径需正确映射到同一对象。留空时为 `https://{Bucket}.{Endpoint 主机名}/{objectKey}`。API 的 `coverUrl` 继续接收和返回完整 HTTPS URL;数据库仅对本平台当前环境资源保存 `/project-covers/...` 未编码路径,域名和环境根目录由后端配置补齐,外链原样保存。创建、PATCH、清除封面及 Fork 的调用方式不变,不允许客户端改为提交相对路径。 + +历史完整平台 URL 及当前环境的 URDF key 均兼容读取,读取不自动迁移。更换域名时,在 `community.oss.legacy-public-base-urls` 配置历史公开基础地址(可带 CDN 基础路径,不含环境根目录、凭据、查询参数及片段),或通过 `OSS_LEGACY_PUBLIC_BASE_URLS` 使用英文逗号分隔。后端按域名、端口和完整路径边界匹配当前环境;带查询参数或片段的封面继续按外链保留。迁移 SQL 与回退步骤见 [资源路径迁移](database.md#资源路径历史数据迁移)。 OSS 控制台配置步骤: @@ -619,7 +621,7 @@ URDF 为可选项目资源,不进入 Gitea,也不生成 Commit。先创建 } ``` -提交签名响应的真实 `batchCode` 和最初的完整清单,不提交 URL、目录前缀或主文件对象键。后端重新识别唯一主文件,生成当前环境、本项目的路径,通过 OSS 核验所有文件存在且实际大小一致后,一次性保存资源记录。成功 data 为更新后的 Project,包含 `urdfMainUrl`、`urdfBaseUrl`;URL 已编码,前端不要再次整体编码。未确认、上传失败、取消或核验失败均不会替换原绑定。 +提交签名响应的真实 `batchCode` 和最初的完整清单,不提交 URL、目录前缀或主文件对象键。后端重新识别唯一主文件,生成当前环境、本项目的路径,通过 OSS 核验所有文件存在且实际大小一致后,一次性保存资源记录。数据库的 `urdf_prefix` 保存 `/urdf/{projectCode}/{batchCode}/`,`urdf_main_object_key` 为此前缀加主文件相对路径,均不含环境根目录和域名;真实 OSS key 仍包含环境根目录。成功 data 为更新后的 Project,包含 `urdfMainUrl`、`urdfBaseUrl` 完整地址;URL 已编码,前端不要再次整体编码。未确认、上传失败、取消或核验失败均不会替换原绑定。 替换重复执行“签名 → 直传 → 确认”,并发以最后一次成功确认生效。签名过期时重新申请新批次并重新上传,不修改旧签名。没有上传会话或后台任务查询接口。 @@ -1257,6 +1259,7 @@ if (result.status !== 'CHAT') { | 500 | CAPTCHA_ERROR | 验证码生成失败,可重新获取并记录 requestId | | 500 | PROJECT_INITIALIZATION_ERROR | 项目初始化或回写未完成,查看 data.projectCode 和工作区,请后端核对 | | 500 | INTERNAL_ERROR | 展示服务异常并保留 requestId | +| 500 | RESOURCE_PATH_INVALID | 历史资源路径格式或所属环境异常,保留 requestId 请后端核对 | | 502 | GITEA_ERROR | 仓库服务异常;写请求先核对结果,避免自动重试 | | 502 | GITEA_INVALID_RESPONSE | 仓库响应无法确认;写请求先核对结果,避免自动重试 | | 503 | GITEA_NOT_CONFIGURED | 后端缺少仓库配置,请后端处理 | diff --git a/docs/database.md b/docs/database.md index a57e45d..73a2be7 100644 --- a/docs/database.md +++ b/docs/database.md @@ -25,10 +25,10 @@ CREATE TABLE project ( owner_user_id BIGINT NOT NULL COMMENT '所属平台用户', name VARCHAR(100) NOT NULL COMMENT '项目名称', description TEXT NOT NULL COMMENT '项目描述', - cover_url VARCHAR(2048) NULL COMMENT '封面图片 HTTPS 地址', - urdf_main_object_key VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' COMMENT 'URDF主文件OSS对象键,不保存完整URL', + cover_url VARCHAR(2048) NULL COMMENT '项目封面,平台资源存环境相对路径,外部图片存完整HTTPS地址', + urdf_main_object_key VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' COMMENT 'URDF主文件环境相对路径,以斜杠开头,不含域名和环境根目录', urdf_files JSON NOT NULL DEFAULT (JSON_ARRAY()) COMMENT 'URDF文件清单数组,元素包含relativePath和sizeBytes,相对路径以urdf_prefix为基准', - urdf_prefix VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' COMMENT 'URDF资源OSS目录前缀,包含环境根目录和上传批次,以斜杠结尾', + urdf_prefix VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' COMMENT 'URDF资源环境相对目录,包含上传批次,以斜杠开头和结尾', project_type VARCHAR(64) NOT NULL COMMENT '配置中的项目类型编码', gitea_repo_id BIGINT NULL COMMENT 'Gitea 仓库标识', gitea_owner VARCHAR(100) NOT NULL COMMENT 'Gitea 仓库所属组织', @@ -91,10 +91,10 @@ CREATE TABLE project_like ( ```sql ALTER TABLE project - ADD COLUMN cover_url VARCHAR(2048) NULL COMMENT '封面图片 HTTPS 地址' AFTER description; + ADD COLUMN cover_url VARCHAR(2048) NULL COMMENT '项目封面,平台资源存环境相对路径,外部图片存完整HTTPS地址' AFTER description; ``` -历史项目的封面为 NULL。字段保存固定公开地址,可以来自 OSS、CDN 或外部 HTTPS 图片;不存储临时签名下载地址。替换、清除封面和删除项目不会自动删除 OSS 对象,Fork 可能共用同一地址。 +历史项目的封面为 NULL。平台封面保存 `/project-covers/...` 环境相对路径;外部 HTTPS 图片保留完整地址。历史完整平台 URL 仍可读取,不保存临时签名下载地址。替换、清除封面和删除项目不会自动删除 OSS 对象,Fork 可能共用同一资源。 ## 已有数据库增加项目 URDF @@ -104,14 +104,193 @@ ALTER TABLE project ALTER TABLE project ADD COLUMN urdf_main_object_key VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' - COMMENT 'URDF主文件OSS对象键,不保存完整URL', + COMMENT 'URDF主文件环境相对路径,以斜杠开头,不含域名和环境根目录', ADD COLUMN urdf_files JSON NOT NULL DEFAULT (JSON_ARRAY()) COMMENT 'URDF文件清单数组,元素包含relativePath和sizeBytes,相对路径以urdf_prefix为基准', ADD COLUMN urdf_prefix VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' - COMMENT 'URDF资源OSS目录前缀,包含环境根目录和上传批次,以斜杠结尾'; + COMMENT 'URDF资源环境相对目录,包含上传批次,以斜杠开头和结尾'; ``` 未上传或已清除时,两个字符串均为 `''`,文件清单为 `[]`。每个项目只绑定一个当前资源包,包内必须恰好有一个 `.urdf`(忽略扩展名大小写)。清单包含全部文件,例如 `[{"relativePath":"urdf/robot.urdf","sizeBytes":1234},{"relativePath":"meshes/base.stl","sizeBytes":5678}]`。 -前缀为 `{环境根目录}/urdf/{projectCode}/{batchCode}/`,主文件对象键为此前缀加主文件相对路径。数据库不保存完整 URL;接口根据 OSS/CDN 配置生成并编码 URL。替换、清除、删除项目不物理删除对象,公开旧链接仍可读取;Fork 复制到新项目的独立目录,不共用源前缀。URDF 不进入 Gitea、不生成 Commit,更新立即影响社区展示。 +存储前缀为 `/urdf/{projectCode}/{batchCode}/`,主文件路径为此前缀加主文件相对路径,不含域名及环境根目录。实际 OSS key 为 `{环境根目录}{存储路径}`,接口根据 OSS/CDN 配置生成并编码 URL。历史带当前环境根目录的 key 兼容读取;其他环境的 key 不会被自动重定位。替换、清除、删除项目不物理删除对象,公开旧链接仍可读取;Fork 复制到新项目的独立目录,不共用源前缀。URDF 不进入 Gitea、不生成 Commit,更新立即影响社区展示。 + +## 资源路径历史数据迁移 + +先升级所有在线及导入写入进程,新版本兼容新旧格式,读取不修改记录。按环境分别执行以下操作;test 根目录为 `vibemake_dev`,prod 为 `vibemake`。OSS 对象不移动,`urdf_files`、`updated_at` 不变。不要在不同环境共用同一业务数据库。 + +迁移前备份数据库,并将候选记录的原值导出至备份目录。以下临时表仅属于当前数据库连接,不能代替持久备份。封面按每个受控公开基础地址分别执行一遍,包含 Bucket 标准域名、当前 CDN 基础地址和 `community.oss.legacy-public-base-urls` 中的历史地址;基础地址不含环境根目录。不使用全局 REPLACE,不转换外链、带参数/片段地址或其他环境路径。 + +### 预览与准备 + +以下命令在同一个 MySQL 8 连接内执行。仅自动转换上传流程生成的规范 ASCII 封面路径,编码形式、非标准目录和异常 URDF 记录列出后跳过,由维护人员核验。数据库比较使用二进制排序规则,避免默认不区分大小写导致误匹配。所有正则的输入和模式均显式指定 `utf8mb4_bin`,避免字段、会话变量和 CONCAT 结果的排序规则推导冲突(1267);仅设置 SET NAMES 不足以消除表达式冲突。这里是字符排序规则,不使用 BINARY 强制转换作为正则参数。 + +```sql +-- resource-path-migration-prepare +SET NAMES utf8mb4 COLLATE utf8mb4_bin; +SET @resource_root = 'vibemake_dev'; +SET @resource_public_base = 'https://linkerbot.oss-cn-beijing.aliyuncs.com'; +SET @cover_prefix = CONCAT(TRIM(TRAILING '/' FROM @resource_public_base), '/', @resource_root, '/project-covers/'); +DROP TEMPORARY TABLE IF EXISTS resource_path_audit; +CREATE TEMPORARY TABLE resource_path_audit AS +SELECT project_code, cover_url AS old_cover_url, + urdf_main_object_key AS old_main, urdf_prefix AS old_prefix, + LEFT(cover_url, CHAR_LENGTH(@cover_prefix)) COLLATE utf8mb4_bin = @cover_prefix COLLATE utf8mb4_bin + AND REGEXP_LIKE(SUBSTRING(cover_url, CHAR_LENGTH(@cover_prefix) + 1) COLLATE utf8mb4_bin, + '^[a-z0-9]{12}([a-z0-9]{20})?/[0-9]{8}/[a-f0-9]{32}[.](jpg|png|webp)$' COLLATE utf8mb4_bin, 'c') AS cover_ok, + REGEXP_LIKE(urdf_prefix COLLATE utf8mb4_bin, + CONCAT('^', @resource_root, '/urdf/', TRIM(project_code), '/[a-f0-9]{32}/$') COLLATE utf8mb4_bin, 'c') + AND LEFT(urdf_main_object_key, CHAR_LENGTH(urdf_prefix)) COLLATE utf8mb4_bin = urdf_prefix COLLATE utf8mb4_bin + AND CHAR_LENGTH(urdf_main_object_key) > CHAR_LENGTH(urdf_prefix) + AND REGEXP_LIKE(urdf_main_object_key COLLATE utf8mb4_bin, '[.]urdf$' COLLATE utf8mb4_bin, 'i') + AND LOCATE('//', urdf_main_object_key) = 0 + AND LOCATE(CHAR(92), urdf_main_object_key) = 0 + AND NOT REGEXP_LIKE(urdf_main_object_key COLLATE utf8mb4_bin, + '(^|/)[.]{1,2}(/|$)|(^|/)[.]git(/|$)|[[:cntrl:]:]' COLLATE utf8mb4_bin, 'i') AS urdf_ok +FROM project; + +DROP TEMPORARY TABLE IF EXISTS resource_path_changes; +CREATE TEMPORARY TABLE resource_path_changes AS +SELECT project_code, old_cover_url, old_main, old_prefix, + CASE WHEN cover_ok THEN CONCAT('/project-covers/', SUBSTRING(old_cover_url, CHAR_LENGTH(@cover_prefix) + 1)) ELSE old_cover_url END AS new_cover_url, + CASE WHEN urdf_ok THEN SUBSTRING(old_main, CHAR_LENGTH(@resource_root) + 1) ELSE old_main END AS new_main, + CASE WHEN urdf_ok THEN SUBSTRING(old_prefix, CHAR_LENGTH(@resource_root) + 1) ELSE old_prefix END AS new_prefix +FROM resource_path_audit WHERE cover_ok OR urdf_ok; + +SELECT COUNT(*) AS candidate_count FROM resource_path_changes; +-- 将下列结果导出并保存;包含迁移前后值,供审计及精确恢复使用。 +SELECT * FROM resource_path_changes ORDER BY project_code; +-- 必须人工核验的封面及 URDF,结果不包含完整外链。 +SELECT project_code, '封面路径非标准或包含参数,请人工核验' AS reason +FROM resource_path_audit +WHERE LEFT(old_cover_url, CHAR_LENGTH(@cover_prefix)) COLLATE utf8mb4_bin = @cover_prefix COLLATE utf8mb4_bin AND NOT cover_ok +UNION ALL +SELECT project_code, 'URDF路径或主文件关系异常,或属于其他环境' AS reason +FROM resource_path_audit +WHERE (old_main <> '' OR old_prefix <> '') AND NOT urdf_ok + AND NOT (LEFT(old_main, 6) = '/urdf/' AND LEFT(old_prefix, 6) = '/urdf/'); +``` + +### 人工执行转换 + +确认备份、预览和异常清单后,在原连接内执行。更新时比较三列原值,避免覆盖预览后的并发修改;数量不足时重新预览,不能删除原值比较条件强制更新。再次运行准备及转换流程时,已转换行不会再次命中。 + +```sql +-- resource-path-migration-apply +START TRANSACTION; +UPDATE project p JOIN resource_path_changes c ON c.project_code = p.project_code +SET p.cover_url = c.new_cover_url, + p.urdf_main_object_key = c.new_main, + p.urdf_prefix = c.new_prefix +WHERE p.cover_url COLLATE utf8mb4_bin <=> c.old_cover_url COLLATE utf8mb4_bin + AND p.urdf_main_object_key COLLATE utf8mb4_bin = c.old_main COLLATE utf8mb4_bin + AND p.urdf_prefix COLLATE utf8mb4_bin = c.old_prefix COLLATE utf8mb4_bin; +COMMIT; +``` + +验证项目列表、详情、AI 卡片及 Fork 的公开 URL 与迁移前指向相同对象。迁移不触发项目重新发布,也不删除任何 OSS 对象。 + +### URDF 更新遇到 1267 时续跑 + +旧脚本如果在 URDF 更新处报 `Illegal mix of collations ... regexp_like`,先前成功的封面更新可能仍未提交。保留原连接,在同一连接执行下面的续跑 SQL;不要重新发起 START TRANSACTION 或修改字段排序规则。末尾 COMMIT 会同时提交该连接中先前成功的更新。prod 将根目录改为 `vibemake`。该脚本不再修改封面,且只匹配未转换的当前环境 URDF。 + +```sql +-- resource-path-migration-resume-urdf +SET NAMES utf8mb4 COLLATE utf8mb4_bin; +SET @resource_root = 'vibemake_dev'; +UPDATE project +SET urdf_main_object_key = SUBSTRING(urdf_main_object_key, CHAR_LENGTH(@resource_root) + 1), + urdf_prefix = SUBSTRING(urdf_prefix, CHAR_LENGTH(@resource_root) + 1) +WHERE REGEXP_LIKE( + urdf_prefix COLLATE utf8mb4_bin, + CONCAT('^', @resource_root, '/urdf/', TRIM(project_code), '/[a-f0-9]{32}/$') COLLATE utf8mb4_bin, + 'c' + ) + AND LEFT(urdf_main_object_key, CHAR_LENGTH(urdf_prefix)) COLLATE utf8mb4_bin = urdf_prefix COLLATE utf8mb4_bin + AND CHAR_LENGTH(urdf_main_object_key) > CHAR_LENGTH(urdf_prefix) + AND REGEXP_LIKE(urdf_main_object_key COLLATE utf8mb4_bin, '[.]urdf$' COLLATE utf8mb4_bin, 'i') + AND LOCATE('//', urdf_main_object_key) = 0 + AND LOCATE(CHAR(92), urdf_main_object_key) = 0 + AND NOT REGEXP_LIKE( + urdf_main_object_key COLLATE utf8mb4_bin, + '(^|/)[.]{1,2}(/|$)|(^|/)[.]git(/|$)|[[:cntrl:]:]' COLLATE utf8mb4_bin, + 'i' + ); +COMMIT; +``` + +### 更新字段中文注释 + +以下 DDL 独立执行,不与数据转换事务混用。字段名、类型、长度及空值约定不变;应用不会自动执行。 + +```sql +ALTER TABLE project + MODIFY COLUMN cover_url VARCHAR(2048) NULL + COMMENT '项目封面,平台资源存环境相对路径,外部图片存完整HTTPS地址', + MODIFY COLUMN urdf_main_object_key VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' + COMMENT 'URDF主文件环境相对路径,以斜杠开头,不含域名和环境根目录', + MODIFY COLUMN urdf_prefix VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' + COMMENT 'URDF资源环境相对目录,包含上传批次,以斜杠开头和结尾'; +``` + +### 回退旧版本 + +先暂停所有资源写入并备份数据库。必须转换当前全部新格式记录,不能仅恢复迁移前备份,否则会丢失升级后的有效写入。回退封面时需要 UTF-8 URL 编码,使用离线生成器准备 SQL 值;工具不连接数据库,也不执行 SQL。 + +```sql +-- 导出查询结果为 JSON Lines(仅结果值,无列头);保持资源写入暂停。 +SELECT JSON_OBJECT('projectCode', project_code, 'coverPath', cover_url) +FROM project WHERE LEFT(cover_url, 16) = '/project-covers/'; +``` + +```bash +python3 scripts/resource-path-rollback.py --root vibemake_dev \ + --public-base-url 'https://linkerbot.oss-cn-beijing.aliyuncs.com' \ + --input /backup/resource-covers.jsonl --output /backup/resource-cover-values.txt +``` + +生成失败时不要继续。新开连接,设置与当前环境一致的根目录,执行以下准备 SQL,再人工执行生成的 `.txt` 中 INSERT 语句。生成器将中文、空格、百分号等正确编码;超出旧字段 2048 字符上限时会拒绝生成,需先修复该项目封面或保留兼容版本。 + +```sql +-- resource-path-rollback-prepare +SET NAMES utf8mb4 COLLATE utf8mb4_bin; +SET @resource_root = 'vibemake_dev'; +DROP TEMPORARY TABLE IF EXISTS resource_cover_rollback; +CREATE TEMPORARY TABLE resource_cover_rollback ( + project_code CHAR(32) NOT NULL PRIMARY KEY, + cover_path VARCHAR(2048) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + full_url VARCHAR(2048) NOT NULL +); +-- 在此执行生成器输出的 INSERT;为空文件表示没有待回退的平台封面。 +``` + +确认映射覆盖所有 `/project-covers/` 记录,再执行以下 SQL。URDF 两列仅在属于本项目且互相匹配时成对恢复前缀;异常记录先修复,不能直接回退代码。 + +```sql +-- resource-path-rollback-apply +START TRANSACTION; +UPDATE project p JOIN resource_cover_rollback c ON c.project_code = p.project_code +SET p.cover_url = c.full_url +WHERE p.cover_url COLLATE utf8mb4_bin = c.cover_path COLLATE utf8mb4_bin; +UPDATE project +SET urdf_main_object_key = CONCAT(@resource_root, urdf_main_object_key), + urdf_prefix = CONCAT(@resource_root, urdf_prefix) +WHERE REGEXP_LIKE(urdf_prefix COLLATE utf8mb4_bin, + CONCAT('^/urdf/', TRIM(project_code), '/[a-f0-9]{32}/$') COLLATE utf8mb4_bin, 'c') + AND LEFT(urdf_main_object_key, CHAR_LENGTH(urdf_prefix)) COLLATE utf8mb4_bin = urdf_prefix COLLATE utf8mb4_bin + AND CHAR_LENGTH(urdf_main_object_key) > CHAR_LENGTH(urdf_prefix) + AND REGEXP_LIKE(urdf_main_object_key COLLATE utf8mb4_bin, '[.]urdf$' COLLATE utf8mb4_bin, 'i') + AND LOCATE('//', urdf_main_object_key) = 0 + AND LOCATE(CHAR(92), urdf_main_object_key) = 0 + AND NOT REGEXP_LIKE(urdf_main_object_key COLLATE utf8mb4_bin, + '(^|/)[.]{1,2}(/|$)|(^|/)[.]git(/|$)|[[:cntrl:]:]' COLLATE utf8mb4_bin, 'i') + AND CHAR_LENGTH(CONCAT(@resource_root, urdf_main_object_key)) <= 512 + AND CHAR_LENGTH(CONCAT(@resource_root, urdf_prefix)) <= 512; +COMMIT; +SELECT project_code FROM project +WHERE LEFT(cover_url, 16) = '/project-covers/' + OR LEFT(urdf_main_object_key, 6) = '/urdf/' OR LEFT(urdf_prefix, 6) = '/urdf/'; +``` + +最后查询必须为空,随后验证旧格式地址并回退应用,再恢复写入。旧版不识别相对路径,禁止只回退代码而跳过数据转换。 diff --git a/scripts/migrate-project-covers.sh b/scripts/migrate-project-covers.sh new file mode 100644 index 0000000..ed61f27 --- /dev/null +++ b/scripts/migrate-project-covers.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +COMMUNITY_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ -f "$COMMUNITY_ROOT/.env" ]]; then + set -a + source "$COMMUNITY_ROOT/.env" + set +a +fi +cd "$COMMUNITY_ROOT" +mvn -q -pl community-import -am -DskipTests package +exec java -Djava.awt.headless=true -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8 \ + -Dlogging.level.root=WARN -Dlogging.level.org.springframework.jdbc=INFO \ + -Dlogging.level.org.apache.http=WARN -Ddebug=false \ + -jar community-import/target/community-import-0.0.1-SNAPSHOT.jar covers "$@" diff --git a/scripts/resource-path-rollback.py b/scripts/resource-path-rollback.py new file mode 100644 index 0000000..e345765 --- /dev/null +++ b/scripts/resource-path-rollback.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""将封面路径导出数据转换为供人工执行的回退 SQL;不连接数据库。""" + +import argparse +import json +from pathlib import Path +import re +from urllib.parse import quote, urlsplit + + +def sql_text(value: str) -> str: + return "CONVERT(0x" + value.encode("utf-8").hex() + " USING utf8mb4)" + + +def generate(rows: list[dict], root: str, public_base: str) -> str: + if not re.fullmatch(r"[A-Za-z0-9_-]+", root): + raise ValueError("环境根目录不合法") + base = urlsplit(public_base) + if (base.scheme != "https" or not base.hostname or base.username is not None + or any(char.isspace() or ord(char) < 32 for char in public_base) + or base.password is not None or "?" in public_base or "#" in public_base + or (base.port is not None and not 1 <= base.port <= 65535)): + raise ValueError("公开基础地址必须是无凭据和参数的 HTTPS 地址") + statements = [] + seen = set() + for row in rows: + code = row.get("projectCode", "") + path = row.get("coverPath", "") + if not isinstance(code, str) or not re.fullmatch(r"[a-z0-9]{12}|[a-z0-9]{32}", code): + raise ValueError("项目编码不合法") + if code in seen: + raise ValueError(f"项目编码重复 projectCode={code}") + seen.add(code) + if (not isinstance(path, str) or not path.startswith("/project-covers/") + or any(ord(char) < 32 or ord(char) == 127 for char in path) + or "\\" in path or ":" in path + or any(part in ("", ".", "..") or part.lower() == ".git" for part in path[1:].split("/"))): + raise ValueError(f"封面存储路径异常 projectCode={code}") + full_url = public_base.rstrip("/") + "/" + root + quote(path, safe="/!$&'()*+,;=@-._~") + if len(full_url) > 2048: + raise ValueError(f"编码后的封面地址超过2048字符,无法回退 projectCode={code}") + statements.append("INSERT INTO resource_cover_rollback (project_code, cover_path, full_url) VALUES (" + + ", ".join(sql_text(value) for value in (code, path, full_url)) + ");") + return "\n".join(statements) + ("\n" if statements else "") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", required=True) + parser.add_argument("--public-base-url", required=True) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + rows = [json.loads(line) for line in args.input.read_text(encoding="utf-8").splitlines() if line.strip()] + result = generate(rows, args.root, args.public_base_url) + # 全部校验成功后才落盘,不输出原始地址或路径。 + args.output.write_text(result, encoding="utf-8") + print(f"已生成回退SQL,记录数={len(rows)};请按数据库文档人工执行") + except (ValueError, TypeError, AttributeError, OSError) as error: + if isinstance(error, json.JSONDecodeError): + parser.exit(1, "输入不是合法的JSON Lines,请使用无列头导出\n") + parser.exit(1, f"回退SQL生成失败:{error.__class__.__name__};检查输入文件、地址、路径及字段长度\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_resource_path_rollback.py b/scripts/tests/test_resource_path_rollback.py new file mode 100644 index 0000000..7f0237e --- /dev/null +++ b/scripts/tests/test_resource_path_rollback.py @@ -0,0 +1,59 @@ +import importlib.util +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + + +spec = importlib.util.spec_from_file_location("rollback", Path(__file__).parents[1] / "resource-path-rollback.py") +rollback = importlib.util.module_from_spec(spec) +spec.loader.exec_module(rollback) + + +class ResourcePathRollbackTest(unittest.TestCase): + def test_preserves_unicode_percent_plus_and_sql_quotes_without_changing_keys(self): + path = "/project-covers/用户/a #?%+'图片.png" + sql = rollback.generate([{"projectCode": "1" * 12, "coverPath": path}], + "vibemake_dev", "https://cdn.example.com/assets/") + values = [bytes.fromhex(value).decode("utf-8") for value in re.findall(r"CONVERT\(0x([0-9a-f]+) USING utf8mb4\)", sql)] + self.assertEqual(values, ["1" * 12, path, + "https://cdn.example.com/assets/vibemake_dev/project-covers/%E7%94%A8%E6%88%B7/" + "a%20%23%3F%25+'%E5%9B%BE%E7%89%87.png"]) + self.assertNotIn(path, sql) + + def test_rejects_duplicates_invalid_paths_and_invalid_origins(self): + valid = {"projectCode": "1" * 12, "coverPath": "/project-covers/a.png"} + with self.assertRaises(ValueError): + rollback.generate([valid, valid], "vibemake", "https://cdn.example.com") + for path in ("/urdf/a.png", "/project-covers/../a.png", "/project-covers/a\\b.png", "/project-covers/a\n.png"): + with self.subTest(path=path), self.assertRaises(ValueError): + rollback.generate([{**valid, "coverPath": path}], "vibemake", "https://cdn.example.com") + for base in ("http://cdn.example.com", "https://user:pass@cdn.example.com", "https://cdn.example.com?token=example"): + with self.subTest(base=base), self.assertRaises(ValueError): + rollback.generate([valid], "vibemake", base) + + def test_rejects_urls_that_would_overflow_old_schema(self): + with self.assertRaises(ValueError): + rollback.generate([{"projectCode": "1" * 12, "coverPath": "/project-covers/" + "中" * 300}], + "vibemake", "https://cdn.example.com") + self.assertEqual(rollback.generate([], "vibemake", "https://cdn.example.com"), "") + + def test_cli_does_not_overwrite_output_or_log_input_on_failure(self): + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "input.jsonl" + output_path = Path(directory) / "output.txt" + input_path.write_text('{"projectCode":"invalid-secret-token","coverPath":"/bad"}', encoding="utf-8") + output_path.write_text("existing output", encoding="utf-8") + result = subprocess.run([sys.executable, str(Path(__file__).parents[1] / "resource-path-rollback.py"), + "--root", "vibemake", "--public-base-url", "https://cdn.example.com", + "--input", str(input_path), "--output", str(output_path)], + capture_output=True, text=True, check=False) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(output_path.read_text(encoding="utf-8"), "existing output") + self.assertNotIn("invalid-secret-token", result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main()