feat(training): release V0.9.1 避障训练与基础策略迁移

training_server/tuning/manager.py

@@ -13,11 +13,20 @@ from copy import deepcopy
 from pathlib import Path
 from typing import Any
 
+from pretrained_sources import PretrainedSources, SourceError
+from task_config import TaskConfigError, validate_task_config
+
+from . import obstacle_scoring
 from .advisor import AdvisorUnavailable, DeepSeekAdvisor
 from .process import GpuLease, ResourceBusyError, terminate_process
 from .schema import (
-    BASE_REWARD_CONFIGURATION,
+    FLAT_TASK,
+    OBSTACLE_TASK,
+    RewardConfigError,
+    base_configuration,
     merge_proposal,
+    task_specs,
+    validate_configuration,
     validate_configuration_constraints,
     validate_constraints,
     validate_proposal,
@@ -51,6 +60,7 @@ class TuningManager:
         data_root: Path,
         lease: GpuLease,
         advisor: Any | None = None,
+        sources: PretrainedSources | None = None,
     ):
         self.trainer_root = trainer_root.expanduser().resolve()
         self.python = python
@@ -66,10 +76,22 @@ class TuningManager:
         self.workers: dict[str, threading.Thread] = {}
         self.processes: dict[str, subprocess.Popen[str]] = {}
         self.cancel_events: dict[str, threading.Event] = {}
+        self.sources = sources
 
     def capability(self) -> dict[str, Any]:
         capability = self.advisor.capability()
-        capability.update({"ready": (self.trainer_root / "scripts" / "evaluate.py").is_file()})
+        capability.update(
+            {
+                "ready": (self.trainer_root / "scripts" / "evaluate.py").is_file(),
+                "pretrainedSources": self.sources.catalog() if self.sources else [],
+                "pretrainedUpload": {
+                    "enabled": self.sources is not None,
+                    "templateId": "go2-legacy47-v1",
+                    "formats": {"pt": 256 * 1024**2, "onnx": 64 * 1024**2},
+                    "endpoint": "/api/training/pretrained-sources/upload",
+                },
+            }
+        )
         return capability
 
     @staticmethod
@@ -85,8 +107,34 @@ class TuningManager:
         mode = payload.get("mode", "automatic")
         if mode not in ("automatic", "approval"):
             raise TuningError("mode 必须是 automatic 或 approval")
-        if payload.get("taskId", "Unitree-Go2-Flat") != "Unitree-Go2-Flat":
-            raise TuningError("第一版只支持 Unitree-Go2-Flat")
+        task_id = payload.get("taskId", "Unitree-Go2-Flat")
+        task_specs(task_id)
+        allowed = {
+            "taskId",
+            "mode",
+            "runName",
+            "gpuIds",
+            "trialCount",
+            "initialIterations",
+            "middleIterations",
+            "finalIterations",
+            "numEnvs",
+            "seed",
+            "evalNumEnvs",
+            "evalSteps",
+            "earlyStopPatience",
+            "objectiveWeights",
+            "fallbackEnabled",
+            "terrainPreset",
+            "terrainParams",
+            "sensorType",
+            "sensorCfg",
+            "customTerrainBoxes",
+            "taskConfig",
+            "pretrainedSourceId",
+        }
+        if payload.keys() - allowed:
+            raise TuningError("未知session字段")
         run_name = payload.get("runName", "auto-tune")
         if not isinstance(run_name, str) or not RUN_NAME.fullmatch(run_name):
             raise TuningError("runName 格式无效")
@@ -105,7 +153,7 @@ class TuningManager:
         rung1 = self._integer(payload, "middleIterations", 900, rung0, 1000000)
         rung2 = self._integer(payload, "finalIterations", 2000, rung1, 1000000)
         config = {
-            "taskId": "Unitree-Go2-Flat",
+            "taskId": task_id,
             "numEnvs": self._integer(payload, "numEnvs", 4096, 1, 16384),
             "seed": self._integer(payload, "seed", 42, 0, 2147483647),
             "runName": run_name,
@@ -117,12 +165,78 @@ class TuningManager:
             "evalSteps": self._integer(payload, "evalSteps", 1000, 10, 100000),
             "earlyStopPatience": self._integer(payload, "earlyStopPatience", 4, 1, 20),
         }
-        objective = validate_objective_weights(
-            payload.get("objectiveWeights", DEFAULT_OBJECTIVE_WEIGHTS)
-        )
+        if task_id == OBSTACLE_TASK:
+            if config["evalSteps"] != obstacle_scoring.STEPS:
+                raise TuningError("避障评估固定1000步，不能调整")
+            objective = deepcopy(obstacle_scoring.WEIGHTS)
+            if "objectiveWeights" in payload and payload["objectiveWeights"] != objective:
+                raise TuningError("避障评估权重固定")
+            task_payload = payload.get("taskConfig", payload)
+            if "taskConfig" in payload:
+                if not isinstance(task_payload, dict) or task_payload.keys() - {
+                    "terrainPreset",
+                    "terrainParams",
+                    "sensorCfg",
+                    "sensorType",
+                    "seed",
+                    "customTerrainBoxes",
+                }:
+                    raise TuningError("taskConfig字段无效")
+                if any(
+                    k in payload
+                    for k in (
+                        "terrainPreset",
+                        "terrainParams",
+                        "sensorCfg",
+                        "sensorType",
+                        "customTerrainBoxes",
+                    )
+                ):
+                    raise TuningError("taskConfig不能与顶层场景配置混用")
+                task_seed = task_payload.get("seed", config["seed"])
+                if (
+                    isinstance(task_seed, bool)
+                    or not isinstance(task_seed, int)
+                    or task_seed != config["seed"]
+                ):
+                    raise TuningError("taskConfig seed必须与session一致")
+            try:
+                config["taskConfig"] = validate_task_config(task_id, task_payload, config["seed"])
+            except TaskConfigError as error:
+                raise TuningError(str(error)) from error
+            base = base_configuration(task_id)
+            base["weights"]["avoidance_weight"] = config["taskConfig"]["sensorCfg"][
+                "avoidanceWeight"
+            ]
+            validate_configuration_constraints(base, {}, task_id)
+        else:
+            if any(
+                k in payload
+                for k in (
+                    "terrainPreset",
+                    "terrainParams",
+                    "sensorCfg",
+                    "sensorType",
+                    "customTerrainBoxes",
+                    "taskConfig",
+                )
+            ):
+                raise TuningError("Flat调参不接受避障场景字段")
+            objective = validate_objective_weights(
+                payload.get("objectiveWeights", DEFAULT_OBJECTIVE_WEIGHTS)
+            )
         fallback = payload.get("fallbackEnabled", False)
         if not isinstance(fallback, bool):
             raise TuningError("fallbackEnabled 必须是布尔值")
+        if "pretrainedSourceId" in payload:
+            if self.sources is None:
+                raise TuningError("服务尚未注册基础策略，请配置--pretrained-sources")
+            try:
+                config["pretrained"] = self.sources.bind(
+                    payload["pretrainedSourceId"], task_id, config.get("taskConfig")
+                )
+            except SourceError as error:
+                raise TuningError(str(error)) from error
         return mode, config, objective, fallback
 
     def create(self, payload: Any) -> dict:
@@ -262,8 +376,36 @@ class TuningManager:
             "--reward-config",
             str(reward_path),
         ]
+        task_path = None
+        if config["taskId"] == OBSTACLE_TASK:
+            task_path = run_dir / "task_config.json"
+            task_path.write_text(
+                json.dumps(config["taskConfig"], allow_nan=False), encoding="utf-8"
+            )
+            command.extend(("--task-config", str(task_path)))
+        pretrained = config.get("pretrained")
+        if pretrained is not None:
+            if self.sources is None:
+                raise TuningError("基础策略快照服务未配置，拒绝随机初始化")
+            self.sources.verify(pretrained)
         if resume_checkpoint is not None:
+            if pretrained is not None:
+                origin_path = resume_checkpoint.parent / "initialization.json"
+                if (
+                    not origin_path.is_file()
+                    or origin_path.is_symlink()
+                    or origin_path.stat().st_size > 64 * 1024
+                ):
+                    raise TuningError("续训checkpoint缺少有效基础策略来源记录，拒绝丢失来源")
+                origin = json.loads(origin_path.read_text(encoding="utf-8"))
+                if (
+                    origin.get("source_id") != pretrained["sourceId"]
+                    or origin.get("artifacts") != pretrained["manifest"]["artifacts"]
+                ):
+                    raise TuningError("续训checkpoint基础来源身份不匹配")
             command.extend(("--resume-checkpoint", str(resume_checkpoint)))
+        elif pretrained is not None:
+            command.extend(self.sources.arguments(pretrained))
         environment = os.environ.copy()
         environment["WANDB_MODE"] = "disabled"
         environment["WANDB_SILENT"] = "true"
@@ -304,6 +446,8 @@ class TuningManager:
             "--gpu-ids",
             json.dumps(config["gpuIds"], separators=(",", ":")),
         ]
+        if task_path is not None:
+            eval_command.extend(("--task-config", str(task_path)))
         self.storage.update_trial(trial_id, state="evaluating", message="正在固定协议评估")
         self.storage.update_session(
             session_id, state="evaluating", message=f"正在评估 trial {trial['number']}"
@@ -316,7 +460,15 @@ class TuningManager:
             raise TuningError(f"评估失败（返回码 {return_code}）")
         evaluation = json.loads(eval_output.read_text(encoding="utf-8"))
         baseline_trial = self.storage.list_trials(session_id)[0]
-        if baseline_trial["evaluation"] is None:
+        if config["taskId"] == OBSTACLE_TASK:
+            expected = obstacle_scoring.protocol(config["taskConfig"], config["evalNumEnvs"])
+            metrics = obstacle_scoring.validate_evaluation(evaluation, expected)
+            baseline = baseline_trial["evaluation"]
+            baseline_metrics = (
+                obstacle_scoring.validate_evaluation(baseline, expected) if baseline else None
+            )
+            scored = obstacle_scoring.score_evaluation(metrics, baseline_metrics)
+        elif baseline_trial["evaluation"] is None:
             scored = {
                 "eligible": True,
                 "score": 0.0,
@@ -394,7 +546,28 @@ class TuningManager:
         return {
             "task": session["config"]["taskId"],
             "objectiveWeights": session["objectiveWeights"],
-            "allowlist": "服务端将验证固定 schema；最多四项修改",
+            "allowlist": {
+                section: {
+                    key: {"min": spec.minimum, "max": spec.maximum} for key, spec in specs.items()
+                }
+                for section, specs in zip(
+                    ("weights", "params"), task_specs(session["config"]["taskId"]), strict=True
+                )
+            },
+            "taskContext": (
+                (
+                    "97维目标导航：48条前视ray，3x16层pitch=[0,-20,-45]deg；仍有侧后/层间/坑盲区。"
+                    if session["config"]["taskConfig"]["sensorCfg"].get("sensorMode")
+                    == "multi_ring_raycast"
+                    else "81维目标导航：32条前视ray仅单高度切片，存在侧后方/矮障碍/跌落盲区。"
+                )
+                + "避障权重对应近障平方惩罚，collision_penalty为非足端>10N接触惩罚；"
+                "target_velocity是真实导航command速度而非reward权重。关注擦碰、绕行、目标到达和动作抖动。"
+                "评估固定seed/1000步/客观权重，不允许修改地图、传感器、起终点、协议或以训练奖励代替指标。"
+            )
+            if session["config"]["taskId"] == OBSTACLE_TASK
+            else "Flat速度跟踪与步态稳定；保留原六目标评估。",
+            "taskConfig": session["config"].get("taskConfig"),
             "parameterConstraints": self.storage.get_control(session["id"])["constraints"],
             "rejectedFeedback": rejected_feedback,
             "trials": [
@@ -410,12 +583,14 @@ class TuningManager:
             ],
         }
 
-    def _fallback_patch(self, previous: dict, index: int) -> dict:
+    def _fallback_patch(self, previous: dict, index: int, task_id="Unitree-Go2-Flat") -> dict:
         names = ("track_linear_velocity", "action_rate_l2", "body_orientation_l2", "foot_slip")
+        if task_id == OBSTACLE_TASK:
+            names = ("avoidance_weight", "collision_penalty")
         name = names[index % len(names)]
         old = previous["weights"][name]
         factor = 1.1 if index % 2 == 0 else 0.9
-        return validate_proposal({"weights": {name: old * factor}}, previous)
+        return validate_proposal({"weights": {name: old * factor}}, previous, task_id=task_id)
 
     def _request_proposal(
         self, session: dict, previous: dict, base_trial_id: str, index: int
@@ -427,7 +602,7 @@ class TuningManager:
             if not session["fallbackEnabled"]:
                 raise AdvisorUnavailable(str(error)) from error
             result = {
-                "patch": self._fallback_patch(previous, index),
+                "patch": self._fallback_patch(previous, index, session["config"]["taskId"]),
                 "rationale": f"Agent 不可用，显式 fallback：{error}",
                 "expectedImpact": {},
                 "confidence": 0.2,
@@ -437,7 +612,9 @@ class TuningManager:
             }
             source = "fallback"
         constraints = self.storage.get_control(session["id"])["constraints"]
-        result["patch"] = validate_proposal(result["patch"], previous, constraints)
+        result["patch"] = validate_proposal(
+            result["patch"], previous, constraints, session["config"]["taskId"]
+        )
         proposal = self.storage.create_proposal(
             session["id"],
             base_trial_id,
@@ -471,11 +648,34 @@ class TuningManager:
                 self.condition.wait(timeout=1.0)
         raise TuningError("session 已取消")
 
+    @staticmethod
+    def _base_configuration(session):
+        base = base_configuration(session["config"]["taskId"])
+        if session["config"]["taskId"] == OBSTACLE_TASK:
+            base["weights"]["avoidance_weight"] = session["config"]["taskConfig"]["sensorCfg"][
+                "avoidanceWeight"
+            ]
+        return base
+
     def _run_session(self, session_id: str, resume: bool, cancel: threading.Event) -> None:
         try:
             session = self.storage.get_session(session_id)
             trials = self.storage.list_trials(session_id)
             if resume:
+                for proposal in self.storage.list_proposals(session_id):
+                    if proposal["state"] == "pending" and self.storage.decide_proposal(
+                        proposal["id"],
+                        "rejected",
+                        "service_restart/recovery_invalidated：服务重启，需重新提案并审批（非用户拒绝）",
+                    ):
+                        self.storage.audit(
+                            session_id,
+                            "recovery_proposal_invalidated",
+                            {
+                                "proposalId": proposal["id"],
+                                "reason": "service_restart/recovery_invalidated",
+                            },
+                        )
                 root = self._session_root(session_id)
                 for interrupted in [trial for trial in trials if trial["state"] == "interrupted"]:
                     run_dir = (root / interrupted["runDir"]).resolve()
@@ -499,7 +699,7 @@ class TuningManager:
                     0,
                     0,
                     session["config"]["rungs"][0],
-                    deepcopy(BASE_REWARD_CONFIGURATION),
+                    self._base_configuration(session),
                     None,
                     baseline_dir,
                 )
@@ -551,7 +751,12 @@ class TuningManager:
                     proposal = self.storage.get_proposal(proposal["id"])
                 self._claim_dispatch(session_id, cancel)
                 constraints = self.storage.get_control(session_id)["constraints"]
-                reward_config = merge_proposal(base["rewardConfig"], proposal["patch"], constraints)
+                reward_config = merge_proposal(
+                    base["rewardConfig"],
+                    proposal["patch"],
+                    constraints,
+                    session["config"]["taskId"],
+                )
                 trial = self.storage.create_trial(
                     session_id,
                     next_number,
@@ -698,7 +903,12 @@ class TuningManager:
                 patch = payload["patch"]
         if feedback is not None and (not isinstance(feedback, str) or len(feedback) > 2000):
             raise TuningError("feedback 无效")
-        patch = validate_proposal(patch, base["rewardConfig"], constraints)
+        patch = validate_proposal(
+            patch,
+            base["rewardConfig"],
+            constraints,
+            self.storage.get_session(session_id)["config"]["taskId"],
+        )
         if not self.storage.decide_proposal(proposal_id, "approved", feedback, patch):
             raise TuningError("proposal 已处理")
         self.storage.audit(
@@ -743,7 +953,12 @@ class TuningManager:
                         continue
                     base = self.storage.get_trial(proposal["baseTrialId"])
                     try:
-                        validate_proposal(proposal["patch"], base["rewardConfig"], constraints)
+                        validate_proposal(
+                            proposal["patch"],
+                            base["rewardConfig"],
+                            constraints,
+                            session["config"]["taskId"],
+                        )
                     except Exception as error:
                         self.storage.decide_proposal(
                             proposal["id"], "rejected", f"参数护栏已变化：{error}"
@@ -771,8 +986,8 @@ class TuningManager:
         revision = payload["revision"]
         if isinstance(revision, bool) or not isinstance(revision, int) or revision < 0:
             raise TuningError("constraints revision 必须是非负整数")
-        constraints = validate_constraints(payload["constraints"])
         session = self.storage.get_session(session_id)
+        constraints = validate_constraints(payload["constraints"], session["config"]["taskId"])
         if session["state"] not in ACTIVE_SESSION_STATES:
             raise TuningError("终态 session 不能修改参数护栏")
         control = self.storage.get_control(session_id)
@@ -784,7 +999,9 @@ class TuningManager:
         else:
             base = self._best_highest_rung(session_id)
         if base is not None:
-            validate_configuration_constraints(base["rewardConfig"], constraints)
+            validate_configuration_constraints(
+                base["rewardConfig"], constraints, session["config"]["taskId"]
+            )
         try:
             updated = self.storage.replace_constraints(session_id, revision, constraints)
         except StorageConflict as error:
@@ -796,7 +1013,12 @@ class TuningManager:
                 continue
             proposal_base = self.storage.get_trial(proposal["baseTrialId"])
             try:
-                validate_proposal(proposal["patch"], proposal_base["rewardConfig"], constraints)
+                validate_proposal(
+                    proposal["patch"],
+                    proposal_base["rewardConfig"],
+                    constraints,
+                    session["config"]["taskId"],
+                )
             except Exception as error:
                 message = f"参数护栏 revision {updated['constraintsRevision']}：{error}"
                 if self.storage.decide_proposal(proposal["id"], "rejected", message):
@@ -914,7 +1136,16 @@ class TuningManager:
         return self.detail(session_id)
 
     def resume(self, session_id: str) -> dict:
+        # Serialize state transition and worker registration against duplicate requests.
+        with self.condition:
+            return self._resume_locked(session_id)
+
+    def _resume_locked(self, session_id: str) -> dict:
         session = self.storage.get_session(session_id)
+        if session["config"].get("pretrained") is not None:
+            if self.sources is None:
+                raise TuningError("基础策略快照服务未配置，无法恢复")
+            self.sources.verify(session["config"]["pretrained"])
         if session["state"] == "paused":
             self.storage.set_run_policy(session_id, "continuous")
             has_pending = any(
@@ -982,8 +1213,11 @@ class TuningManager:
             raise TuningError("最佳策略文件不存在")
         return path
 
-    def preset_config(self, preset_id: str) -> dict:
-        return self.storage.get_preset(preset_id)["rewardConfig"]
+    def preset_config(self, preset_id: str, task_id: str = FLAT_TASK) -> dict:
+        preset = self.storage.get_preset(preset_id)
+        if preset["taskId"] != task_id:
+            raise RewardConfigError("奖励 preset 来源任务与训练任务不匹配")
+        return validate_configuration(preset["rewardConfig"], task_id)
 
     def test_agent(self) -> dict:
         try: