cover_url、urdf_main_object_key、urdf_prefix只存相对路径,不包含域名+环境根目录
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
COMMUNITY_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
if [[ -f "$COMMUNITY_ROOT/.env" ]]; then
|
||||
set -a
|
||||
source "$COMMUNITY_ROOT/.env"
|
||||
set +a
|
||||
fi
|
||||
cd "$COMMUNITY_ROOT"
|
||||
mvn -q -pl community-import -am -DskipTests package
|
||||
exec java -Djava.awt.headless=true -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8 \
|
||||
-Dlogging.level.root=WARN -Dlogging.level.org.springframework.jdbc=INFO \
|
||||
-Dlogging.level.org.apache.http=WARN -Ddebug=false \
|
||||
-jar community-import/target/community-import-0.0.1-SNAPSHOT.jar covers "$@"
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
"""将封面路径导出数据转换为供人工执行的回退 SQL;不连接数据库。"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
|
||||
def sql_text(value: str) -> str:
|
||||
return "CONVERT(0x" + value.encode("utf-8").hex() + " USING utf8mb4)"
|
||||
|
||||
|
||||
def generate(rows: list[dict], root: str, public_base: str) -> str:
|
||||
if not re.fullmatch(r"[A-Za-z0-9_-]+", root):
|
||||
raise ValueError("环境根目录不合法")
|
||||
base = urlsplit(public_base)
|
||||
if (base.scheme != "https" or not base.hostname or base.username is not None
|
||||
or any(char.isspace() or ord(char) < 32 for char in public_base)
|
||||
or base.password is not None or "?" in public_base or "#" in public_base
|
||||
or (base.port is not None and not 1 <= base.port <= 65535)):
|
||||
raise ValueError("公开基础地址必须是无凭据和参数的 HTTPS 地址")
|
||||
statements = []
|
||||
seen = set()
|
||||
for row in rows:
|
||||
code = row.get("projectCode", "")
|
||||
path = row.get("coverPath", "")
|
||||
if not isinstance(code, str) or not re.fullmatch(r"[a-z0-9]{12}|[a-z0-9]{32}", code):
|
||||
raise ValueError("项目编码不合法")
|
||||
if code in seen:
|
||||
raise ValueError(f"项目编码重复 projectCode={code}")
|
||||
seen.add(code)
|
||||
if (not isinstance(path, str) or not path.startswith("/project-covers/")
|
||||
or any(ord(char) < 32 or ord(char) == 127 for char in path)
|
||||
or "\\" in path or ":" in path
|
||||
or any(part in ("", ".", "..") or part.lower() == ".git" for part in path[1:].split("/"))):
|
||||
raise ValueError(f"封面存储路径异常 projectCode={code}")
|
||||
full_url = public_base.rstrip("/") + "/" + root + quote(path, safe="/!$&'()*+,;=@-._~")
|
||||
if len(full_url) > 2048:
|
||||
raise ValueError(f"编码后的封面地址超过2048字符,无法回退 projectCode={code}")
|
||||
statements.append("INSERT INTO resource_cover_rollback (project_code, cover_path, full_url) VALUES ("
|
||||
+ ", ".join(sql_text(value) for value in (code, path, full_url)) + ");")
|
||||
return "\n".join(statements) + ("\n" if statements else "")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", required=True)
|
||||
parser.add_argument("--public-base-url", required=True)
|
||||
parser.add_argument("--input", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
rows = [json.loads(line) for line in args.input.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
result = generate(rows, args.root, args.public_base_url)
|
||||
# 全部校验成功后才落盘,不输出原始地址或路径。
|
||||
args.output.write_text(result, encoding="utf-8")
|
||||
print(f"已生成回退SQL,记录数={len(rows)};请按数据库文档人工执行")
|
||||
except (ValueError, TypeError, AttributeError, OSError) as error:
|
||||
if isinstance(error, json.JSONDecodeError):
|
||||
parser.exit(1, "输入不是合法的JSON Lines,请使用无列头导出\n")
|
||||
parser.exit(1, f"回退SQL生成失败:{error.__class__.__name__};检查输入文件、地址、路径及字段长度\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,59 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
spec = importlib.util.spec_from_file_location("rollback", Path(__file__).parents[1] / "resource-path-rollback.py")
|
||||
rollback = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(rollback)
|
||||
|
||||
|
||||
class ResourcePathRollbackTest(unittest.TestCase):
|
||||
def test_preserves_unicode_percent_plus_and_sql_quotes_without_changing_keys(self):
|
||||
path = "/project-covers/用户/a #?%+'图片.png"
|
||||
sql = rollback.generate([{"projectCode": "1" * 12, "coverPath": path}],
|
||||
"vibemake_dev", "https://cdn.example.com/assets/")
|
||||
values = [bytes.fromhex(value).decode("utf-8") for value in re.findall(r"CONVERT\(0x([0-9a-f]+) USING utf8mb4\)", sql)]
|
||||
self.assertEqual(values, ["1" * 12, path,
|
||||
"https://cdn.example.com/assets/vibemake_dev/project-covers/%E7%94%A8%E6%88%B7/"
|
||||
"a%20%23%3F%25+'%E5%9B%BE%E7%89%87.png"])
|
||||
self.assertNotIn(path, sql)
|
||||
|
||||
def test_rejects_duplicates_invalid_paths_and_invalid_origins(self):
|
||||
valid = {"projectCode": "1" * 12, "coverPath": "/project-covers/a.png"}
|
||||
with self.assertRaises(ValueError):
|
||||
rollback.generate([valid, valid], "vibemake", "https://cdn.example.com")
|
||||
for path in ("/urdf/a.png", "/project-covers/../a.png", "/project-covers/a\\b.png", "/project-covers/a\n.png"):
|
||||
with self.subTest(path=path), self.assertRaises(ValueError):
|
||||
rollback.generate([{**valid, "coverPath": path}], "vibemake", "https://cdn.example.com")
|
||||
for base in ("http://cdn.example.com", "https://user:pass@cdn.example.com", "https://cdn.example.com?token=example"):
|
||||
with self.subTest(base=base), self.assertRaises(ValueError):
|
||||
rollback.generate([valid], "vibemake", base)
|
||||
|
||||
def test_rejects_urls_that_would_overflow_old_schema(self):
|
||||
with self.assertRaises(ValueError):
|
||||
rollback.generate([{"projectCode": "1" * 12, "coverPath": "/project-covers/" + "中" * 300}],
|
||||
"vibemake", "https://cdn.example.com")
|
||||
self.assertEqual(rollback.generate([], "vibemake", "https://cdn.example.com"), "")
|
||||
|
||||
def test_cli_does_not_overwrite_output_or_log_input_on_failure(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
input_path = Path(directory) / "input.jsonl"
|
||||
output_path = Path(directory) / "output.txt"
|
||||
input_path.write_text('{"projectCode":"invalid-secret-token","coverPath":"/bad"}', encoding="utf-8")
|
||||
output_path.write_text("existing output", encoding="utf-8")
|
||||
result = subprocess.run([sys.executable, str(Path(__file__).parents[1] / "resource-path-rollback.py"),
|
||||
"--root", "vibemake", "--public-base-url", "https://cdn.example.com",
|
||||
"--input", str(input_path), "--output", str(output_path)],
|
||||
capture_output=True, text=True, check=False)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(output_path.read_text(encoding="utf-8"), "existing output")
|
||||
self.assertNotIn("invalid-secret-token", result.stdout + result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user