feat(training): release V0.9.1 避障训练与基础策略迁移

training_server/tuning/schema.py

@@ -68,6 +68,32 @@ BASE_REWARD_CONFIGURATION: dict[str, dict[str, float]] = {
 }
 
 
+FLAT_TASK = "Unitree-Go2-Flat"
+OBSTACLE_TASK = "Unitree-Go2-ObstacleAvoidance"
+OBSTACLE_WEIGHT_SPECS = {
+    "avoidance_weight": NumericSpec(0.5, 5.0, 2.0, False),
+    "collision_penalty": NumericSpec(-10.0, -0.5, -5.0, False),
+    "action_smoothness": NumericSpec(-0.05, -0.001, -0.05, False),
+}
+OBSTACLE_PARAMETER_SPECS = {"target_velocity": NumericSpec(0.3, 1.2, 0.6, False)}
+
+
+def task_specs(task_id=FLAT_TASK):
+    if task_id == FLAT_TASK:
+        return WEIGHT_SPECS, PARAMETER_SPECS
+    if task_id == OBSTACLE_TASK:
+        return OBSTACLE_WEIGHT_SPECS, OBSTACLE_PARAMETER_SPECS
+    raise RewardConfigError(f"只支持Flat或Obstacle调参任务：{task_id}")
+
+
+def base_configuration(task_id=FLAT_TASK):
+    weights, params = task_specs(task_id)
+    return {
+        "weights": {k: v.default for k, v in weights.items()},
+        "params": {k: v.default for k, v in params.items()},
+    }
+
+
 def _number(name: str, value: Any, spec: NumericSpec) -> float:
     if isinstance(value, bool) or not isinstance(value, (int, float)):
         raise RewardConfigError(f"{name} 必须是数值")
@@ -87,19 +113,22 @@ def _mapping(value: Any, name: str) -> Mapping[str, Any]:
     return value
 
 
-def _cross_validate(config: Mapping[str, Mapping[str, float]]) -> None:
+def _cross_validate(config: Mapping[str, Mapping[str, float]], task_id=FLAT_TASK) -> None:
+    if task_id == OBSTACLE_TASK:
+        return
     params = config["params"]
     if params["pose.walking_threshold"] >= params["pose.running_threshold"]:
         raise RewardConfigError("pose.walking_threshold 必须小于 pose.running_threshold")
 
 
-def _path_spec(path: str) -> tuple[str, str, NumericSpec]:
+def _path_spec(path: str, task_id=FLAT_TASK) -> tuple[str, str, NumericSpec]:
+    weights, params = task_specs(task_id)
     if path.startswith("weights."):
         section, name = "weights", path.removeprefix("weights.")
-        spec = WEIGHT_SPECS.get(name)
+        spec = weights.get(name)
     elif path.startswith("params."):
         section, name = "params", path.removeprefix("params.")
-        spec = PARAMETER_SPECS.get(name)
+        spec = params.get(name)
     else:
         section, name, spec = "", "", None
     if spec is None:
@@ -107,16 +136,16 @@ def _path_spec(path: str) -> tuple[str, str, NumericSpec]:
     return section, name, spec
 
 
-def validate_constraints(value: Any) -> dict[str, dict[str, float | str]]:
+def validate_constraints(value: Any, task_id=FLAT_TASK) -> dict[str, dict[str, float | str]]:
     """Validate sparse per-session range/fixed safety constraints."""
     root = _mapping(value, "constraints")
-    if len(root) > len(WEIGHT_SPECS) + len(PARAMETER_SPECS):
+    if len(root) > sum(len(specs) for specs in task_specs(task_id)):
         raise RewardConfigError("constraints 数量超过白名单参数总数")
     result: dict[str, dict[str, float | str]] = {}
     for raw_path, raw_constraint in root.items():
         if not isinstance(raw_path, str):
             raise RewardConfigError("constraint path 必须是字符串")
-        _, _, spec = _path_spec(raw_path)
+        _, _, spec = _path_spec(raw_path, task_id)
         constraint = _mapping(raw_constraint, raw_path)
         kind = constraint.get("kind")
         if kind == "fixed":
@@ -137,12 +166,12 @@ def validate_constraints(value: Any) -> dict[str, dict[str, float | str]]:
     return result
 
 
-def validate_configuration_constraints(value: Any, constraints: Any) -> None:
+def validate_configuration_constraints(value: Any, constraints: Any, task_id=FLAT_TASK) -> None:
     """Ensure a complete reward configuration satisfies every session constraint."""
-    config = validate_configuration(value)
-    checked = validate_constraints(constraints)
+    config = validate_configuration(value, task_id)
+    checked = validate_constraints(constraints, task_id)
     for path, constraint in checked.items():
-        section, name, _ = _path_spec(path)
+        section, name, _ = _path_spec(path, task_id)
         current = config[section][name]
         if constraint["kind"] == "fixed":
             if current != constraint["value"]:
@@ -155,36 +184,37 @@ def validate_configuration_constraints(value: Any, constraints: Any) -> None:
             )
 
 
-def validate_configuration(value: Any) -> dict[str, dict[str, float]]:
+def validate_configuration(value: Any, task_id=FLAT_TASK) -> dict[str, dict[str, float]]:
     """Validate a complete configuration and reject missing/unknown fields."""
     root = _mapping(value, "rewardConfig")
     if set(root) != {"weights", "params"}:
         raise RewardConfigError("rewardConfig 只能包含 weights 和 params")
+    weights, params = task_specs(task_id)
     raw_weights = _mapping(root["weights"], "weights")
     raw_params = _mapping(root["params"], "params")
-    if set(raw_weights) != set(WEIGHT_SPECS):
+    if set(raw_weights) != set(weights):
         raise RewardConfigError("weights 必须完整且不能包含未知奖励项")
-    if set(raw_params) != set(PARAMETER_SPECS):
+    if set(raw_params) != set(params):
         raise RewardConfigError("params 必须完整且不能包含未知参数")
     config = {
         "weights": {
             name: _number(f"weights.{name}", raw_weights[name], spec)
-            for name, spec in WEIGHT_SPECS.items()
+            for name, spec in weights.items()
         },
         "params": {
-            name: _number(f"params.{name}", raw_params[name], spec)
-            for name, spec in PARAMETER_SPECS.items()
+            name: _number(f"params.{name}", raw_params[name], spec) for name, spec in params.items()
         },
     }
-    _cross_validate(config)
+    _cross_validate(config, task_id)
     return config
 
 
 def validate_proposal(
-    value: Any, previous: Any, constraints: Any | None = None
+    value: Any, previous: Any, constraints: Any | None = None, task_id=FLAT_TASK
 ) -> dict[str, dict[str, float]]:
     """Validate a sparse Agent patch relative to a complete previous config and guardrails."""
-    current = validate_configuration(previous)
+    weights, params = task_specs(task_id)
+    current = validate_configuration(previous, task_id)
     root = _mapping(value, "proposal")
     if not set(root).issubset({"weights", "params"}):
         raise RewardConfigError("proposal 只能包含 weights 和 params")
@@ -194,8 +224,8 @@ def validate_proposal(
         raise RewardConfigError("proposal 至少需要一项修改")
     if len(raw_weights) + len(raw_params) > MAX_PROPOSAL_CHANGES:
         raise RewardConfigError(f"proposal 每轮最多修改 {MAX_PROPOSAL_CHANGES} 项")
-    unknown_weights = set(raw_weights) - set(WEIGHT_SPECS)
-    unknown_params = set(raw_params) - set(PARAMETER_SPECS)
+    unknown_weights = set(raw_weights) - set(weights)
+    unknown_params = set(raw_params) - set(params)
     if unknown_weights:
         raise RewardConfigError(f"未知奖励项：{', '.join(sorted(unknown_weights))}")
     if unknown_params:
@@ -203,7 +233,7 @@ def validate_proposal(
 
     patch: dict[str, dict[str, float]] = {"weights": {}, "params": {}}
     for name, raw in raw_weights.items():
-        value_number = _number(f"weights.{name}", raw, WEIGHT_SPECS[name])
+        value_number = _number(f"weights.{name}", raw, weights[name])
         old = current["weights"][name]
         if old != 0.0 and value_number != 0.0:
             ratio = abs(value_number / old)
@@ -216,7 +246,7 @@ def validate_proposal(
             raise RewardConfigError(f"weights.{name} 没有发生变化")
         patch["weights"][name] = value_number
     for name, raw in raw_params.items():
-        value_number = _number(f"params.{name}", raw, PARAMETER_SPECS[name])
+        value_number = _number(f"params.{name}", raw, params[name])
         old = current["params"][name]
         ratio = abs(value_number / old)
         if ratio < MIN_CHANGE_RATIO or ratio > MAX_CHANGE_RATIO:
@@ -230,26 +260,39 @@ def validate_proposal(
     candidate = deepcopy(current)
     candidate["weights"].update(patch["weights"])
     candidate["params"].update(patch["params"])
-    _cross_validate(candidate)
+    _cross_validate(candidate, task_id)
     if constraints is not None:
-        validate_configuration_constraints(candidate, constraints)
+        validate_configuration_constraints(candidate, constraints, task_id)
     return patch
 
 
 def merge_proposal(
-    previous: Any, proposal: Any, constraints: Any | None = None
+    previous: Any, proposal: Any, constraints: Any | None = None, task_id=FLAT_TASK
 ) -> dict[str, dict[str, float]]:
-    current = validate_configuration(previous)
-    patch = validate_proposal(proposal, current, constraints)
+    current = validate_configuration(previous, task_id)
+    patch = validate_proposal(proposal, current, constraints, task_id)
     merged = deepcopy(current)
     merged["weights"].update(patch["weights"])
     merged["params"].update(patch["params"])
-    return validate_configuration(merged)
+    return validate_configuration(merged, task_id)
 
 
-def apply_reward_configuration(env_cfg: Any, value: Any) -> None:
+def apply_reward_configuration(env_cfg: Any, value: Any, task_id=FLAT_TASK) -> None:
     """Apply a validated full config to a fresh mjlab environment config."""
-    config = validate_configuration(value)
+    config = validate_configuration(value, task_id)
+    if task_id == OBSTACLE_TASK:
+        from mjlab.managers import RewardTermCfg
+
+        contact = env_cfg.terminations["illegal_contact"]
+        env_cfg.rewards["obstacle_collision"] = RewardTermCfg(
+            func=contact.func,
+            params=deepcopy(contact.params),
+            weight=config["weights"]["collision_penalty"],
+        )
+        env_cfg.rewards["obstacle_proximity"].weight = -config["weights"]["avoidance_weight"]
+        env_cfg.rewards["action_rate_l2"].weight = config["weights"]["action_smoothness"]
+        env_cfg.commands["twist"].speed = config["params"]["target_velocity"]
+        return
     for name, weight in config["weights"].items():
         if name not in env_cfg.rewards:
             raise RewardConfigError(f"环境缺少奖励项：{name}")