cover_url、urdf_main_object_key、urdf_prefix只存相对路径,不包含域名+环境根目录

This commit is contained in:
lxp
2026-10-19 14:53:06 +08:00
parent 8a3ecabdc0
commit f8f2e95dbe
34 changed files with 1700 additions and 58 deletions
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
COMMUNITY_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
if [[ -f "$COMMUNITY_ROOT/.env" ]]; then
set -a
source "$COMMUNITY_ROOT/.env"
set +a
fi
cd "$COMMUNITY_ROOT"
mvn -q -pl community-import -am -DskipTests package
exec java -Djava.awt.headless=true -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8 \
-Dlogging.level.root=WARN -Dlogging.level.org.springframework.jdbc=INFO \
-Dlogging.level.org.apache.http=WARN -Ddebug=false \
-jar community-import/target/community-import-0.0.1-SNAPSHOT.jar covers "$@"
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env python3
"""将封面路径导出数据转换为供人工执行的回退 SQL;不连接数据库。"""
import argparse
import json
from pathlib import Path
import re
from urllib.parse import quote, urlsplit
def sql_text(value: str) -> str:
return "CONVERT(0x" + value.encode("utf-8").hex() + " USING utf8mb4)"
def generate(rows: list[dict], root: str, public_base: str) -> str:
if not re.fullmatch(r"[A-Za-z0-9_-]+", root):
raise ValueError("环境根目录不合法")
base = urlsplit(public_base)
if (base.scheme != "https" or not base.hostname or base.username is not None
or any(char.isspace() or ord(char) < 32 for char in public_base)
or base.password is not None or "?" in public_base or "#" in public_base
or (base.port is not None and not 1 <= base.port <= 65535)):
raise ValueError("公开基础地址必须是无凭据和参数的 HTTPS 地址")
statements = []
seen = set()
for row in rows:
code = row.get("projectCode", "")
path = row.get("coverPath", "")
if not isinstance(code, str) or not re.fullmatch(r"[a-z0-9]{12}|[a-z0-9]{32}", code):
raise ValueError("项目编码不合法")
if code in seen:
raise ValueError(f"项目编码重复 projectCode={code}")
seen.add(code)
if (not isinstance(path, str) or not path.startswith("/project-covers/")
or any(ord(char) < 32 or ord(char) == 127 for char in path)
or "\\" in path or ":" in path
or any(part in ("", ".", "..") or part.lower() == ".git" for part in path[1:].split("/"))):
raise ValueError(f"封面存储路径异常 projectCode={code}")
full_url = public_base.rstrip("/") + "/" + root + quote(path, safe="/!$&'()*+,;=@-._~")
if len(full_url) > 2048:
raise ValueError(f"编码后的封面地址超过2048字符,无法回退 projectCode={code}")
statements.append("INSERT INTO resource_cover_rollback (project_code, cover_path, full_url) VALUES ("
+ ", ".join(sql_text(value) for value in (code, path, full_url)) + ");")
return "\n".join(statements) + ("\n" if statements else "")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", required=True)
parser.add_argument("--public-base-url", required=True)
parser.add_argument("--input", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
try:
rows = [json.loads(line) for line in args.input.read_text(encoding="utf-8").splitlines() if line.strip()]
result = generate(rows, args.root, args.public_base_url)
# 全部校验成功后才落盘,不输出原始地址或路径。
args.output.write_text(result, encoding="utf-8")
print(f"已生成回退SQL,记录数={len(rows)};请按数据库文档人工执行")
except (ValueError, TypeError, AttributeError, OSError) as error:
if isinstance(error, json.JSONDecodeError):
parser.exit(1, "输入不是合法的JSON Lines,请使用无列头导出\n")
parser.exit(1, f"回退SQL生成失败:{error.__class__.__name__};检查输入文件、地址、路径及字段长度\n")
if __name__ == "__main__":
main()
@@ -0,0 +1,59 @@
import importlib.util
from pathlib import Path
import re
import subprocess
import sys
import tempfile
import unittest
spec = importlib.util.spec_from_file_location("rollback", Path(__file__).parents[1] / "resource-path-rollback.py")
rollback = importlib.util.module_from_spec(spec)
spec.loader.exec_module(rollback)
class ResourcePathRollbackTest(unittest.TestCase):
def test_preserves_unicode_percent_plus_and_sql_quotes_without_changing_keys(self):
path = "/project-covers/用户/a #?%+'图片.png"
sql = rollback.generate([{"projectCode": "1" * 12, "coverPath": path}],
"vibemake_dev", "https://cdn.example.com/assets/")
values = [bytes.fromhex(value).decode("utf-8") for value in re.findall(r"CONVERT\(0x([0-9a-f]+) USING utf8mb4\)", sql)]
self.assertEqual(values, ["1" * 12, path,
"https://cdn.example.com/assets/vibemake_dev/project-covers/%E7%94%A8%E6%88%B7/"
"a%20%23%3F%25+'%E5%9B%BE%E7%89%87.png"])
self.assertNotIn(path, sql)
def test_rejects_duplicates_invalid_paths_and_invalid_origins(self):
valid = {"projectCode": "1" * 12, "coverPath": "/project-covers/a.png"}
with self.assertRaises(ValueError):
rollback.generate([valid, valid], "vibemake", "https://cdn.example.com")
for path in ("/urdf/a.png", "/project-covers/../a.png", "/project-covers/a\\b.png", "/project-covers/a\n.png"):
with self.subTest(path=path), self.assertRaises(ValueError):
rollback.generate([{**valid, "coverPath": path}], "vibemake", "https://cdn.example.com")
for base in ("http://cdn.example.com", "https://user:pass@cdn.example.com", "https://cdn.example.com?token=example"):
with self.subTest(base=base), self.assertRaises(ValueError):
rollback.generate([valid], "vibemake", base)
def test_rejects_urls_that_would_overflow_old_schema(self):
with self.assertRaises(ValueError):
rollback.generate([{"projectCode": "1" * 12, "coverPath": "/project-covers/" + "中" * 300}],
"vibemake", "https://cdn.example.com")
self.assertEqual(rollback.generate([], "vibemake", "https://cdn.example.com"), "")
def test_cli_does_not_overwrite_output_or_log_input_on_failure(self):
with tempfile.TemporaryDirectory() as directory:
input_path = Path(directory) / "input.jsonl"
output_path = Path(directory) / "output.txt"
input_path.write_text('{"projectCode":"invalid-secret-token","coverPath":"/bad"}', encoding="utf-8")
output_path.write_text("existing output", encoding="utf-8")
result = subprocess.run([sys.executable, str(Path(__file__).parents[1] / "resource-path-rollback.py"),
"--root", "vibemake", "--public-base-url", "https://cdn.example.com",
"--input", str(input_path), "--output", str(output_path)],
capture_output=True, text=True, check=False)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(output_path.read_text(encoding="utf-8"), "existing output")
self.assertNotIn("invalid-secret-token", result.stdout + result.stderr)
if __name__ == "__main__":
unittest.main()