From 63d67a645b73eacbf70faeb8b3a3035a55fc8eea Mon Sep 17 00:00:00 2001 From: cen617-code <1057290604@qq.com> Date: Thu, 3 Sep 2026 16:25:53 +0800 Subject: [PATCH] =?UTF-8?q?feat(tuning):=20release=20V0.8.2=20Agent=20?= =?UTF-8?q?=E7=95=8C=E9=9D=A2=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 4 +- package.json | 2 +- training_server/README.md | 10 +- training_server/server.py | 45 +- training_server/tests/test_tuning.py | 46 +- training_server/tests/test_tuning_manager.py | 155 +++ training_server/tuning/manager.py | 340 +++++- training_server/tuning/schema.py | 76 +- training_server/tuning/storage.py | 172 ++- web_platform/README.md | 2 +- .../src/training/LocalTrainingClient.test.ts | 19 +- .../src/training/LocalTrainingClient.ts | 38 + .../src/training/LocalTrainingPanel.test.tsx | 57 + .../src/training/LocalTrainingPanel.tsx | 44 +- web_platform/src/training/types.ts | 22 +- .../src/tuning/AgentDecisionTimeline.test.tsx | 141 +++ .../src/tuning/AgentDecisionTimeline.tsx | 478 ++++++++ .../tuning/MetricsComparisonBoard.test.tsx | 116 ++ .../src/tuning/MetricsComparisonBoard.tsx | 495 ++++++++ .../src/tuning/RewardConfigDiffEditor.tsx | 61 + .../src/tuning/ScalarRingBuffer.test.ts | 23 + web_platform/src/tuning/ScalarRingBuffer.ts | 96 ++ web_platform/src/tuning/TuningApp.test.tsx | 7 +- web_platform/src/tuning/TuningApp.tsx | 1029 +++++------------ web_platform/src/tuning/TuningConsole.tsx | 27 + .../src/tuning/TuningControlToolbar.test.tsx | 48 + .../src/tuning/TuningControlToolbar.tsx | 523 +++++++++ web_platform/src/tuning/TuningLeaderboard.tsx | 95 ++ .../src/tuning/TuningSessionRail.test.tsx | 119 ++ web_platform/src/tuning/TuningSessionRail.tsx | 344 ++++++ web_platform/src/tuning/domain.ts | 218 ++++ web_platform/src/tuning/tuningStore.test.ts | 153 +++ web_platform/src/tuning/tuningStore.ts | 558 +++++++++ web_platform/src/tuning/useTuningPolling.ts | 93 ++ web_platform/tuning.html | 2 +- 35 files changed, 4847 insertions(+), 811 deletions(-) create mode 100644 web_platform/src/tuning/AgentDecisionTimeline.test.tsx create mode 100644 web_platform/src/tuning/AgentDecisionTimeline.tsx create mode 100644 web_platform/src/tuning/MetricsComparisonBoard.test.tsx create mode 100644 web_platform/src/tuning/MetricsComparisonBoard.tsx create mode 100644 web_platform/src/tuning/RewardConfigDiffEditor.tsx create mode 100644 web_platform/src/tuning/ScalarRingBuffer.test.ts create mode 100644 web_platform/src/tuning/ScalarRingBuffer.ts create mode 100644 web_platform/src/tuning/TuningConsole.tsx create mode 100644 web_platform/src/tuning/TuningControlToolbar.test.tsx create mode 100644 web_platform/src/tuning/TuningControlToolbar.tsx create mode 100644 web_platform/src/tuning/TuningLeaderboard.tsx create mode 100644 web_platform/src/tuning/TuningSessionRail.test.tsx create mode 100644 web_platform/src/tuning/TuningSessionRail.tsx create mode 100644 web_platform/src/tuning/domain.ts create mode 100644 web_platform/src/tuning/tuningStore.test.ts create mode 100644 web_platform/src/tuning/tuningStore.ts create mode 100644 web_platform/src/tuning/useTuningPolling.ts diff --git a/package-lock.json b/package-lock.json index 22785ca6..639559aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mujoco-web-platform", - "version": "0.8.1", + "version": "0.8.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mujoco-web-platform", - "version": "0.8.1", + "version": "0.8.2", "license": "Apache-2.0", "dependencies": { "@monaco-editor/react": "^4.7.0", diff --git a/package.json b/package.json index 9f0e54ca..c97d62f7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mujoco-web-platform", - "version": "0.8.1", + "version": "0.8.2", "description": "基于 MuJoCo WebAssembly 的本地机器人仿真与控制平台", "private": true, "type": "module", diff --git a/training_server/README.md b/training_server/README.md index 09535196..377721ac 100644 --- a/training_server/README.md +++ b/training_server/README.md @@ -57,7 +57,7 @@ python training_server/server.py \ 在主工作台连接训练服务后,点击“打开自调参 Agent 工作台”。默认预算为 12 个唯一配置:所有配置先训练 300 iterations,前 4 名续训到 900,前 2 名续训到 2000;默认使用 GPU 0 和 4096 个并行环境。首次使用建议先降低为 256–512 environments 做 smoke test。 -固定评估使用站立、前进/侧移、转向和组合命令以及 3 个固定 seed。最终分数不直接使用可被权重放大的总 reward,而由速度跟踪 35%、动作平滑 20%、姿态稳定 15%、减少跌倒 15%、足端滑移 10%、能耗 5% 的权重无关指标组成。跌倒率高于基线 2% 或速度误差恶化超过 5% 的 trial 不晋级。逐轮审批模式会自动运行基线,之后每条 Agent 建议都等待批准、修改后批准或拒绝反馈。 +固定评估使用站立、前进/侧移、转向和组合命令以及 3 个固定 seed。最终分数不直接使用可被权重放大的总 reward,而由速度跟踪 35%、动作平滑 20%、姿态稳定 15%、减少跌倒 15%、足端滑移 10%、能耗 5% 的权重无关指标组成。跌倒率高于基线 2% 或速度误差恶化超过 5% 的 trial 不晋级。逐轮审批模式会自动运行基线,之后每条 Agent 建议都等待批准、修改后批准或拒绝反馈;运行中的 session 也可在逐轮审批和全自动之间切换,切到全自动时会批准当前待处理建议。 最佳结果保存为不可变 preset,可在普通训练面板的“奖励配置”中选择,也可导出 JSON;不会覆盖仓库里的 Python 默认奖励配置。 @@ -71,12 +71,16 @@ python training_server/server.py \ - `GET /api/tuning/capabilities`、`POST /api/tuning/agent/test`:检查/测试 Agent; - `GET|POST /api/tuning/sessions`、`GET|DELETE /api/tuning/sessions/{id}`:列出、创建、查询、停止 session; - `POST /api/tuning/sessions/{id}/pause|resume`:暂停后续调度或恢复; +- `POST /api/tuning/sessions/{id}/mode`:运行时切换 `automatic`/`approval` 模式; +- `PUT /api/tuning/sessions/{id}/constraints`:以 revision CAS 保存参数固定值/工程上下限,服务端在 Agent、fallback 与人工修改三条路径统一强制; +- `POST /api/tuning/sessions/{id}/step`:发放且只消费一个 Trial 调度令牌,完成训练与固定评估后重新暂停; +- `POST /api/tuning/sessions/{id}/rollback`:把同 Session 内已完成且通过安全门槛的 Trial 设为非破坏性后续基准,可同时验证其 checkpoint; - `POST /api/tuning/sessions/{id}/proposals/{proposalId}/approve|reject`:审批、修改或拒绝建议; -- `GET /api/tuning/sessions/{id}/trials/{trialId}/metrics`:查询降采样 scalar; +- `GET /api/tuning/sessions/{id}/trials/{trialId}/metrics?afterStep=N`:查询降采样或增量 scalar; - `GET /api/tuning/sessions/{id}/artifacts/best/policy.onnx`:下载最佳策略; - `GET /api/tuning/presets`:列出可供普通训练复用的最佳奖励 preset。 -普通任务状态在服务重启后丢失,但日志、checkpoint 和 ONNX 保留在 `training_server/rl/logs/rsl_rl/`;调参状态及产物持久化在 `logs/auto_tuning/`。API 只接收 32 位资源 ID,不接收客户端文件路径;奖励 patch 受到名称、符号、上下界、每轮最多 4 项及 `0.5×–2×` 变化率校验。 +普通任务状态在服务重启后丢失,但日志、checkpoint 和 ONNX 保留在 `training_server/rl/logs/rsl_rl/`;调参状态、调度令牌、参数护栏、回滚基准及产物持久化在 `logs/auto_tuning/`。候选配置数可在 1–100 间设置(包含基线配置,仍受连续无提升早停约束)。API 只接收 32 位资源 ID,不接收客户端文件路径;奖励 patch 受到名称、符号、上下界、Session 护栏、每轮最多 4 项及 `0.5×–2×` 变化率校验。 ## 测试 diff --git a/training_server/server.py b/training_server/server.py index 12a78fe6..f12524cd 100644 --- a/training_server/server.py +++ b/training_server/server.py @@ -27,6 +27,8 @@ from urllib.parse import parse_qs, unquote, urlsplit from tuning.manager import TuningError, TuningManager from tuning.process import GpuLease, ResourceBusyError +from tuning.schema import RewardConfigError +from tuning.scoring import EvaluationError VERSION = "0.4.0" # 浏览器当前 ONNX 运行时只实现 Go2 的 47→12 部署契约;其他任务须由服务启动参数显式放行。 @@ -498,7 +500,7 @@ class TrainingRequestHandler(BaseHTTPRequestHandler): self._json(HTTPStatus.NOT_FOUND, {"error": "调参 session、trial 或 proposal 不存在"}) elif isinstance(error, ResourceBusyError): self._json(HTTPStatus.CONFLICT, {"error": str(error)}) - elif isinstance(error, TuningError): + elif isinstance(error, (TuningError, RewardConfigError, EvaluationError)): self._json(HTTPStatus.BAD_REQUEST, {"error": str(error)}) else: self._json( @@ -552,7 +554,7 @@ class TrainingRequestHandler(BaseHTTPRequestHandler): self._ensure_origin() self.send_response(HTTPStatus.NO_CONTENT) self._cors() - self.send_header("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS") + self.send_header("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") self.send_header("Access-Control-Allow-Headers", "Authorization, Content-Type") self.send_header("Access-Control-Max-Age", "600") self.end_headers() @@ -591,14 +593,18 @@ class TrainingRequestHandler(BaseHTTPRequestHandler): tags = [tag for value in query.get("tags", []) for tag in value.split(",") if tag] try: max_points = int(query.get("maxPoints", ["1000"])[0]) + after_raw = query.get("afterStep", [None])[0] + after_step = int(after_raw) if after_raw is not None else None except ValueError as error: - raise TuningError("maxPoints 必须是整数") from error + raise TuningError("maxPoints/afterStep 必须是整数") from error if not 10 <= max_points <= 5000: raise TuningError("maxPoints 必须在 10–5000 之间") + if after_step is not None and after_step < -1: + raise TuningError("afterStep 不能小于 -1") self._json( HTTPStatus.OK, self.tuning_manager.metrics( - match.group(1), match.group(2), tags or None, max_points + match.group(1), match.group(2), tags or None, max_points, after_step ), ) return @@ -640,6 +646,22 @@ class TrainingRequestHandler(BaseHTTPRequestHandler): ) self._json(HTTPStatus.ACCEPTED, action(match.group(1))) return + match = re.fullmatch(r"/api/tuning/sessions/([0-9a-f]{32})/(step|rollback)", path) + if match: + action = ( + self.tuning_manager.step + if match.group(2) == "step" + else self.tuning_manager.rollback + ) + self._json(HTTPStatus.ACCEPTED, action(match.group(1), self._payload())) + return + match = re.fullmatch(r"/api/tuning/sessions/([0-9a-f]{32})/mode", path) + if match: + self._json( + HTTPStatus.ACCEPTED, + self.tuning_manager.set_mode(match.group(1), self._payload()), + ) + return match = re.fullmatch( r"/api/tuning/sessions/([0-9a-f]{32})/proposals/([0-9a-f]{32})/(approve|reject)", path, @@ -658,6 +680,21 @@ class TrainingRequestHandler(BaseHTTPRequestHandler): except Exception as error: self._error(error) + def do_PUT(self) -> None: + try: + self._ensure_request() + path = urlsplit(self.path).path + match = re.fullmatch(r"/api/tuning/sessions/([0-9a-f]{32})/constraints", path) + if match: + self._json( + HTTPStatus.ACCEPTED, + self.tuning_manager.set_constraints(match.group(1), self._payload()), + ) + return + raise ApiError(HTTPStatus.NOT_FOUND, "接口不存在") + except Exception as error: + self._error(error) + def do_DELETE(self) -> None: try: self._ensure_request() diff --git a/training_server/tests/test_tuning.py b/training_server/tests/test_tuning.py index 88d06a31..e988ed27 100644 --- a/training_server/tests/test_tuning.py +++ b/training_server/tests/test_tuning.py @@ -12,6 +12,7 @@ from tuning.schema import ( # noqa: E402 RewardConfigError, merge_proposal, validate_configuration, + validate_constraints, validate_proposal, ) from tuning.scoring import ( # noqa: E402 @@ -19,7 +20,7 @@ from tuning.scoring import ( # noqa: E402 EvaluationError, score_evaluation, ) -from tuning.storage import TuningStorage # noqa: E402 +from tuning.storage import StorageConflict, TuningStorage # noqa: E402 class RewardSchemaTest(unittest.TestCase): @@ -69,6 +70,33 @@ class RewardSchemaTest(unittest.TestCase): BASE_REWARD_CONFIGURATION, ) + def test_session_constraints_reject_unknown_out_of_range_and_fixed_changes(self): + constraints = validate_constraints( + { + "weights.track_linear_velocity": {"kind": "fixed", "value": 1.0}, + "params.foot_gait.period": {"kind": "range", "min": 0.5, "max": 0.7}, + } + ) + validate_proposal( + {"params": {"foot_gait.period": 0.65}}, + BASE_REWARD_CONFIGURATION, + constraints, + ) + with self.assertRaisesRegex(RewardConfigError, "已固定"): + validate_proposal( + {"weights": {"track_linear_velocity": 1.1}}, + BASE_REWARD_CONFIGURATION, + constraints, + ) + with self.assertRaisesRegex(RewardConfigError, "工程锁定范围"): + validate_proposal( + {"params": {"foot_gait.period": 0.75}}, + BASE_REWARD_CONFIGURATION, + constraints, + ) + with self.assertRaisesRegex(RewardConfigError, "未知参数约束"): + validate_constraints({"weights.not_allowed": {"kind": "fixed", "value": 1.0}}) + class ScoringTest(unittest.TestCase): baseline = { @@ -174,8 +202,24 @@ class StorageTest(unittest.TestCase): self.assertEqual(sampled[0]["step"], 0) self.assertEqual(sampled[-1]["step"], 99) self.assertIn(50.0, [point["value"] for point in sampled]) + control = self.storage.replace_constraints( + session["id"], + 0, + {"weights.pose": {"kind": "range", "min": 0.5, "max": 1.5}}, + ) + self.assertEqual(control["constraintsRevision"], 1) + with self.assertRaises(StorageConflict): + self.storage.replace_constraints(session["id"], 0, {}) + self.storage.grant_dispatch_token(session["id"]) + with self.assertRaises(StorageConflict): + self.storage.grant_dispatch_token(session["id"]) + self.assertTrue(self.storage.use_dispatch_token(session["id"])) + self.assertFalse(self.storage.use_dispatch_token(session["id"])) + incremental = self.storage.metrics(trial["id"], max_points=100, after_step=90)[0] + self.assertEqual(incremental["points"][0]["step"], 91) reopened = TuningStorage(self.storage.path) self.assertEqual(reopened.get_session(session["id"])["mode"], "approval") + self.assertEqual(reopened.get_control(session["id"])["constraintsRevision"], 1) def test_recovery_marks_inflight_records(self): session = self.storage.create_session( diff --git a/training_server/tests/test_tuning_manager.py b/training_server/tests/test_tuning_manager.py index 61b66e37..a90d0c7d 100644 --- a/training_server/tests/test_tuning_manager.py +++ b/training_server/tests/test_tuning_manager.py @@ -148,6 +148,8 @@ class TuningManagerTest(unittest.TestCase): self.fail("session did not wait for approval") proposal = detail["proposals"][-1] patch = {"weights": {"pose": 1.1}, "params": {}} + with self.assertRaisesRegex(Exception, "feedback"): + self.manager.approve(session["id"], proposal["id"], {"feedback": {}}) approved = self.manager.approve( session["id"], proposal["id"], {"feedback": "ok", "patch": patch} ) @@ -155,6 +157,34 @@ class TuningManagerTest(unittest.TestCase): self.manager.cancel(session["id"]) self.assertEqual(self.wait_terminal(session["id"])["state"], "cancelled") + def test_runtime_mode_switch_auto_approves_pending_proposal(self): + session = self.manager.create(self.payload("approval")) + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + detail = self.manager.detail(session["id"]) + if detail["state"] == "awaiting_approval": + break + time.sleep(0.01) + else: + self.fail("session did not wait for approval") + changed = self.manager.set_mode(session["id"], {"mode": "automatic"}) + self.assertEqual(changed["mode"], "automatic") + self.assertEqual(changed["proposals"][-1]["state"], "approved") + completed = self.wait_terminal(session["id"]) + self.assertEqual(completed["state"], "succeeded", completed["message"]) + + def test_trial_count_is_user_configurable(self): + payload = self.payload() + payload["trialCount"] = 1 + _, config, _, _ = self.manager.parse_create(payload) + self.assertEqual(config["trialCount"], 1) + payload["trialCount"] = 100 + _, config, _, _ = self.manager.parse_create(payload) + self.assertEqual(config["trialCount"], 100) + payload["trialCount"] = 101 + with self.assertRaisesRegex(Exception, "trialCount"): + self.manager.parse_create(payload) + def test_resume_discards_only_interrupted_trial_and_continues(self): mode, config, objective, fallback = self.manager.parse_create(self.payload()) session = self.manager.storage.create_session(mode, config, objective, fallback) @@ -178,8 +208,10 @@ class TuningManagerTest(unittest.TestCase): "trial-001-rung-0", ) self.manager.storage.update_trial(interrupted["id"], state="interrupted") + self.manager.storage.reset_step_gate(session["id"]) self.manager.storage.update_session(session["id"], state="interrupted") self.manager.resume(session["id"]) + self.assertEqual(self.manager.storage.get_control(session["id"])["runPolicy"], "continuous") completed = self.wait_terminal(session["id"]) self.assertEqual(completed["state"], "succeeded", completed["message"]) self.assertNotIn(interrupted["id"], [trial["id"] for trial in completed["trials"]]) @@ -190,6 +222,129 @@ class TuningManagerTest(unittest.TestCase): self.manager.parse_create({"taskId": "Other"}) self.assertEqual(self.manager.test_agent()["model"], "fake") + def test_pause_closes_persistent_dispatch_gate_at_trial_boundary(self): + mode, config, objective, fallback = self.manager.parse_create(self.payload()) + session = self.manager.storage.create_session(mode, config, objective, fallback) + self.manager.storage.update_session(session["id"], state="running") + self.manager.storage.grant_dispatch_token(session["id"]) + + paused = self.manager.pause(session["id"]) + + self.assertEqual(paused["state"], "paused") + self.assertEqual(paused["control"]["runPolicy"], "step") + self.assertEqual(paused["control"]["dispatchTokens"], 0) + + def test_step_token_executes_exactly_one_trial_then_pauses(self): + session = self.manager.create(self.payload("approval")) + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + detail = self.manager.detail(session["id"]) + if detail["state"] == "awaiting_approval": + break + time.sleep(0.01) + else: + self.fail("session did not wait for approval") + baseline_count = len([trial for trial in detail["trials"] if trial["state"] == "completed"]) + stepped = self.manager.step(session["id"], {"count": 1}) + self.assertEqual(stepped["control"]["runPolicy"], "step") + self.assertEqual(stepped["control"]["dispatchTokens"], 1) + proposal = stepped["proposals"][-1] + self.manager.approve(session["id"], proposal["id"], {}) + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + detail = self.manager.detail(session["id"]) + completed_count = len( + [trial for trial in detail["trials"] if trial["state"] == "completed"] + ) + if detail["state"] == "paused" and completed_count == baseline_count + 1: + break + time.sleep(0.01) + else: + self.fail("single-step trial did not pause at the next boundary") + time.sleep(0.05) + self.assertEqual( + len( + [ + trial + for trial in self.manager.detail(session["id"])["trials"] + if trial["state"] == "completed" + ] + ), + baseline_count + 1, + ) + self.manager.cancel(session["id"]) + self.assertEqual(self.wait_terminal(session["id"])["state"], "cancelled") + + def test_constraints_are_revisioned_and_enforced_during_approval(self): + session = self.manager.create(self.payload("approval")) + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + detail = self.manager.detail(session["id"]) + if detail["state"] == "awaiting_approval": + break + time.sleep(0.01) + else: + self.fail("session did not wait for approval") + proposal = detail["proposals"][-1] + constrained = self.manager.set_constraints( + session["id"], + { + "revision": 0, + "constraints": {"weights.track_linear_velocity": {"kind": "fixed", "value": 1.0}}, + }, + ) + self.assertEqual(constrained["control"]["constraintsRevision"], 1) + with self.assertRaisesRegex(Exception, "已固定"): + self.manager.approve( + session["id"], + proposal["id"], + {"patch": {"weights": {"track_linear_velocity": 1.1}, "params": {}}}, + ) + with self.assertRaisesRegex(Exception, "revision"): + self.manager.set_constraints(session["id"], {"revision": 0, "constraints": {}}) + self.manager.approve(session["id"], proposal["id"], {}) + self.manager.cancel(session["id"]) + self.assertEqual(self.wait_terminal(session["id"])["state"], "cancelled") + + def test_rollback_uses_safe_completed_trial_as_next_proposal_base(self): + session = self.manager.create(self.payload("approval")) + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + detail = self.manager.detail(session["id"]) + if detail["state"] == "awaiting_approval": + break + time.sleep(0.01) + else: + self.fail("session did not wait for approval") + baseline = detail["trials"][0] + old_proposal = detail["proposals"][-1] + rolled_back = self.manager.rollback( + session["id"], {"trialId": baseline["id"], "checkpoint": True} + ) + self.assertEqual(rolled_back["state"], "paused") + self.assertEqual(rolled_back["control"]["activeBaseTrialId"], baseline["id"]) + self.assertEqual( + next(item for item in rolled_back["proposals"] if item["id"] == old_proposal["id"])[ + "state" + ], + "rejected", + ) + self.manager.step(session["id"], {"count": 1}) + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + detail = self.manager.detail(session["id"]) + if ( + detail["state"] == "awaiting_approval" + and detail["proposals"][-1]["id"] != old_proposal["id"] + ): + break + time.sleep(0.01) + else: + self.fail("rollback base did not produce a replacement proposal") + self.assertEqual(detail["proposals"][-1]["baseTrialId"], baseline["id"]) + self.manager.cancel(session["id"]) + self.assertEqual(self.wait_terminal(session["id"])["state"], "cancelled") + if __name__ == "__main__": unittest.main() diff --git a/training_server/tuning/manager.py b/training_server/tuning/manager.py index a63eeba7..d2306012 100644 --- a/training_server/tuning/manager.py +++ b/training_server/tuning/manager.py @@ -18,10 +18,12 @@ from .process import GpuLease, ResourceBusyError, terminate_process from .schema import ( BASE_REWARD_CONFIGURATION, merge_proposal, + validate_configuration_constraints, + validate_constraints, validate_proposal, ) from .scoring import DEFAULT_OBJECTIVE_WEIGHTS, score_evaluation, validate_objective_weights -from .storage import TuningStorage, now_iso +from .storage import StorageConflict, TuningStorage, now_iso from .study import OptunaStudies from .tensorboard import ingest_scalars @@ -98,7 +100,7 @@ class TuningManager: ) ): raise TuningError("gpuIds 必须是非空非负整数数组") - trial_count = self._integer(payload, "trialCount", 12, 4, 20) + trial_count = self._integer(payload, "trialCount", 12, 1, 100) rung0 = self._integer(payload, "initialIterations", 300, 1, 1000000) rung1 = self._integer(payload, "middleIterations", 900, rung0, 1000000) rung2 = self._integer(payload, "finalIterations", 2000, rung1, 1000000) @@ -159,6 +161,15 @@ class TuningManager: session["trials"] = self.storage.list_trials(session_id) session["proposals"] = self.storage.list_proposals(session_id) session["audit"] = self.storage.audit_events(session_id) + control = self.storage.get_control(session_id) + current = next( + (trial for trial in session["trials"] if trial["id"] == session["currentTrialId"]), + None, + ) + control["effectiveAfterCurrent"] = bool( + current and current["state"] in {"training", "evaluating"} + ) + session["control"] = control return session def list(self) -> list[dict]: @@ -274,8 +285,8 @@ class TuningManager: policy = run_dir / "policy.onnx" if return_code != 0 or checkpoint is None or not policy.is_file(): raise TuningError(f"训练失败(返回码 {return_code})或缺少 checkpoint/policy.onnx") - self._wait_if_paused(session_id, self.cancel_events[session_id]) - + # 暂停只关闭后续 Trial 调度门;已经开始的 Trial 必须连同固定评估一起 + # 完成,避免把一次单步令牌错误地消耗在半个 Trial 上。 eval_output = run_dir / "evaluation.json" eval_command = [ self.python, @@ -332,6 +343,9 @@ class TuningManager: score=scored["score"], eligible=scored["eligible"], ) + best = self._best_highest_rung(session_id) + if best is not None: + self.storage.update_session(session_id, best_trial_id=best["id"]) try: optuna_number = self.studies.record( session_id, @@ -363,6 +377,13 @@ class TuningManager: default=None, ) + def _best_highest_rung(self, session_id: str) -> dict | None: + for rung in (2, 1, 0): + best = self._best(session_id, rung=rung) + if best is not None: + return best + return None + def _proposal_context(self, session: dict) -> dict: trials = self.storage.list_trials(session["id"])[-12:] rejected_feedback = [ @@ -374,6 +395,7 @@ class TuningManager: "task": session["config"]["taskId"], "objectiveWeights": session["objectiveWeights"], "allowlist": "服务端将验证固定 schema;最多四项修改", + "parameterConstraints": self.storage.get_control(session["id"])["constraints"], "rejectedFeedback": rejected_feedback, "trials": [ { @@ -414,6 +436,8 @@ class TuningManager: "model": "optuna-fallback", } source = "fallback" + constraints = self.storage.get_control(session["id"])["constraints"] + result["patch"] = validate_proposal(result["patch"], previous, constraints) proposal = self.storage.create_proposal( session["id"], base_trial_id, @@ -487,7 +511,10 @@ class TuningManager: if t["rung"] == 0 and t["state"] == "completed" ] next_number = len({t["number"] for t in completed_rung0}) - best_score = max((t["score"] or 0.0 for t in completed_rung0), default=0.0) + best_score = max( + (t["score"] if t["score"] is not None else float("-inf") for t in completed_rung0), + default=0.0, + ) no_improve = 0 while ( next_number < session["config"]["trialCount"] @@ -495,12 +522,19 @@ class TuningManager: ): if cancel.is_set(): raise TuningError("session 已取消") - self._wait_if_paused(session_id, cancel) - base = self._best(session_id, rung=0) or completed_rung0[0] + self._wait_for_dispatch(session_id, cancel) + control = self.storage.get_control(session_id) + base = ( + self.storage.get_trial(control["activeBaseTrialId"]) + if control["activeBaseTrialId"] + else self._best(session_id, rung=0) or completed_rung0[0] + ) proposal = self._request_proposal( session, base["rewardConfig"], base["id"], next_number ) - if session["mode"] == "approval": + # 模式允许在 session 运行期间切换,因此每次决策都读取最新持久化值。 + current_mode = self.storage.get_session(session_id)["mode"] + if current_mode == "approval": self.storage.update_session( session_id, state="awaiting_approval", message="等待批准 Agent 建议" ) @@ -515,8 +549,9 @@ class TuningManager: else: self.storage.decide_proposal(proposal["id"], "approved", "自动模式") proposal = self.storage.get_proposal(proposal["id"]) - self._wait_if_paused(session_id, cancel) - reward_config = merge_proposal(base["rewardConfig"], proposal["patch"]) + self._claim_dispatch(session_id, cancel) + constraints = self.storage.get_control(session_id)["constraints"] + reward_config = merge_proposal(base["rewardConfig"], proposal["patch"], constraints) trial = self.storage.create_trial( session_id, next_number, @@ -527,8 +562,16 @@ class TuningManager: f"trial-{next_number:03d}-rung-0", ) result = self._execute_trial(session, trial) - if result["eligible"] and (result["score"] or -999) > best_score + 0.01: - best_score = result["score"] + if control["activeBaseTrialId"]: + self.storage.set_active_base(session_id, None) + self._pause_after_step(session_id) + result_score = result["score"] + if ( + result["eligible"] + and result_score is not None + and result_score > best_score + 0.01 + ): + best_score = result_score no_improve = 0 else: no_improve += 1 @@ -537,13 +580,17 @@ class TuningManager: # Promote top configurations; each new rung resumes its own previous checkpoint. for rung in (1, 2): - self._wait_if_paused(session_id, cancel) previous = [ t for t in self.storage.list_trials(session_id) if t["rung"] == rung - 1 and t["state"] == "completed" and t["eligible"] ] - previous.sort(key=lambda t: t["score"] or -999, reverse=True) + previous.sort( + key=lambda trial: ( + trial["score"] if trial["score"] is not None else float("-inf") + ), + reverse=True, + ) promoted_numbers = { trial["number"] for trial in self.storage.list_trials(session_id) @@ -554,6 +601,7 @@ class TuningManager: continue if cancel.is_set(): raise TuningError("session 已取消") + self._claim_dispatch(session_id, cancel) root = self._session_root(session_id) checkpoint = root / parent["checkpointPath"] trial = self.storage.create_trial( @@ -566,12 +614,9 @@ class TuningManager: f"trial-{parent['number']:03d}-rung-{rung}", ) self._execute_trial(session, trial, checkpoint) + self._pause_after_step(session_id) - best = ( - self._best(session_id, rung=2) - or self._best(session_id, rung=1) - or self._best(session_id, rung=0) - ) + best = self._best_highest_rung(session_id) if best is None: raise TuningError("没有通过安全门槛的 trial") preset_name = f"{session['config']['runName']}-{session_id[:8]}" @@ -602,10 +647,42 @@ class TuningManager: self.workers.pop(session_id, None) self.processes.pop(session_id, None) - def _wait_if_paused(self, session_id: str, cancel: threading.Event) -> None: + def _wait_for_dispatch(self, session_id: str, cancel: threading.Event) -> None: + """Wait at a scheduler boundary without consuming a one-shot token.""" with self.condition: - while self.storage.get_session(session_id)["state"] == "paused" and not cancel.is_set(): + while not cancel.is_set(): + session = self.storage.get_session(session_id) + control = self.storage.get_control(session_id) + if session["state"] == "paused": + self.condition.wait(timeout=1.0) + continue + if control["runPolicy"] == "continuous" or control["dispatchTokens"] > 0: + return + self.storage.update_session( + session_id, + state="paused", + message="单步 Trial 已完成;等待下一个调度令牌", + ) + self.storage.audit(session_id, "step_gate_waiting", {}) self.condition.wait(timeout=1.0) + raise TuningError("session 已取消") + + def _claim_dispatch(self, session_id: str, cancel: threading.Event) -> None: + while not cancel.is_set(): + self._wait_for_dispatch(session_id, cancel) + if self.storage.use_dispatch_token(session_id): + return + raise TuningError("session 已取消") + + def _pause_after_step(self, session_id: str) -> None: + control = self.storage.get_control(session_id) + if control["runPolicy"] == "step" and control["dispatchTokens"] == 0: + self.storage.update_session( + session_id, + state="paused", + message="单步 Trial 已完成;后续调度已暂停", + ) + self.storage.audit(session_id, "step_trial_completed", {}) def approve(self, session_id: str, proposal_id: str, payload: Any) -> dict: proposal = self.storage.get_proposal(proposal_id) @@ -613,11 +690,15 @@ class TuningManager: raise TuningError("proposal 不属于该 session") patch = proposal["patch"] feedback = None + base = self.storage.get_trial(proposal["baseTrialId"]) + constraints = self.storage.get_control(session_id)["constraints"] if isinstance(payload, dict): feedback = payload.get("feedback") if "patch" in payload: - base = self.storage.get_trial(proposal["baseTrialId"]) - patch = validate_proposal(payload["patch"], base["rewardConfig"]) + patch = payload["patch"] + if feedback is not None and (not isinstance(feedback, str) or len(feedback) > 2000): + raise TuningError("feedback 无效") + patch = validate_proposal(patch, base["rewardConfig"], constraints) if not self.storage.decide_proposal(proposal_id, "approved", feedback, patch): raise TuningError("proposal 已处理") self.storage.audit( @@ -642,22 +723,213 @@ class TuningManager: self.condition.notify_all() return self.detail(session_id) + def set_mode(self, session_id: str, payload: Any) -> dict: + if not isinstance(payload, dict) or payload.get("mode") not in ("automatic", "approval"): + raise TuningError("mode 必须是 automatic 或 approval") + mode = payload["mode"] + with self.condition: + session = self.storage.get_session(session_id) + if session["state"] not in RUNNING_STATES | {"awaiting_approval", "paused"}: + raise TuningError("当前状态不能切换运行模式") + previous = session["mode"] + if previous == mode: + return self.detail(session_id) + self.storage.update_session(session_id, mode=mode) + approved_ids = [] + if mode == "automatic": + constraints = self.storage.get_control(session_id)["constraints"] + for proposal in self.storage.list_proposals(session_id): + if proposal["state"] != "pending": + continue + base = self.storage.get_trial(proposal["baseTrialId"]) + try: + validate_proposal(proposal["patch"], base["rewardConfig"], constraints) + except Exception as error: + self.storage.decide_proposal( + proposal["id"], "rejected", f"参数护栏已变化:{error}" + ) + continue + if self.storage.decide_proposal( + proposal["id"], "approved", "运行时切换为全自动模式" + ): + approved_ids.append(proposal["id"]) + if session["state"] == "awaiting_approval": + self.storage.update_session( + session_id, state="running", message="已切换为全自动模式,继续调参" + ) + self.storage.audit( + session_id, + "session_mode_changed", + {"from": previous, "to": mode, "autoApprovedProposalIds": approved_ids}, + ) + self.condition.notify_all() + return self.detail(session_id) + + def set_constraints(self, session_id: str, payload: Any) -> dict: + if not isinstance(payload, dict) or set(payload) != {"revision", "constraints"}: + raise TuningError("参数护栏请求必须包含 revision 与 constraints") + revision = payload["revision"] + if isinstance(revision, bool) or not isinstance(revision, int) or revision < 0: + raise TuningError("constraints revision 必须是非负整数") + constraints = validate_constraints(payload["constraints"]) + session = self.storage.get_session(session_id) + if session["state"] not in ACTIVE_SESSION_STATES: + raise TuningError("终态 session 不能修改参数护栏") + control = self.storage.get_control(session_id) + base = None + if control["activeBaseTrialId"]: + base = self.storage.get_trial(control["activeBaseTrialId"]) + elif session["currentTrialId"]: + base = self.storage.get_trial(session["currentTrialId"]) + else: + base = self._best_highest_rung(session_id) + if base is not None: + validate_configuration_constraints(base["rewardConfig"], constraints) + try: + updated = self.storage.replace_constraints(session_id, revision, constraints) + except StorageConflict as error: + raise ResourceBusyError(str(error)) from error + + rejected = [] + for proposal in self.storage.list_proposals(session_id): + if proposal["state"] != "pending": + continue + proposal_base = self.storage.get_trial(proposal["baseTrialId"]) + try: + validate_proposal(proposal["patch"], proposal_base["rewardConfig"], constraints) + except Exception as error: + message = f"参数护栏 revision {updated['constraintsRevision']}:{error}" + if self.storage.decide_proposal(proposal["id"], "rejected", message): + rejected.append(proposal["id"]) + self.storage.audit( + session_id, + "constraints_updated", + { + "revision": updated["constraintsRevision"], + "paths": sorted(constraints), + "rejectedProposalIds": rejected, + }, + ) + if rejected: + with self.condition: + self.condition.notify_all() + return self.detail(session_id) + + def step(self, session_id: str, payload: Any) -> dict: + if not isinstance(payload, dict) or payload.get("count", 1) != 1: + raise TuningError("单步调度一次只能发放 1 个 Trial 令牌") + session = self.storage.get_session(session_id) + if session["state"] not in {"paused", "awaiting_approval"}: + raise TuningError("请先暂停或等待 Proposal 审批,再执行单步 Trial") + control = self.storage.get_control(session_id) + if control["dispatchTokens"] > 0: + raise TuningError("已有未消费的单步 Trial 令牌") + try: + self.storage.grant_dispatch_token(session_id) + except StorageConflict as error: + raise ResourceBusyError(str(error)) from error + if session["state"] == "paused": + has_pending = any( + proposal["state"] == "pending" + for proposal in self.storage.list_proposals(session_id) + ) + self.storage.update_session( + session_id, + state="awaiting_approval" if has_pending else "running", + message="单步令牌已就绪;请审批 Proposal" + if has_pending + else "已授权执行一个 Trial", + ) + self.storage.audit(session_id, "step_token_granted", {"count": 1}) + with self.condition: + self.condition.notify_all() + return self.detail(session_id) + + def rollback(self, session_id: str, payload: Any) -> dict: + if not isinstance(payload, dict): + raise TuningError("rollback 请求体必须是对象") + session = self.storage.get_session(session_id) + if session["state"] not in {"paused", "awaiting_approval"}: + raise TuningError("回滚只能在安全暂停或等待审批时执行") + target_id = payload.get("trialId") + if payload.get("target") == "best": + target_id = session["bestTrialId"] or (self._best_highest_rung(session_id) or {}).get( + "id" + ) + if not isinstance(target_id, str): + raise TuningError("rollback 必须指定 trialId 或 target=best") + target = self.storage.get_trial(target_id) + if target["sessionId"] != session_id: + raise TuningError("rollback Trial 不属于该 session") + if target["state"] != "completed" or not target["eligible"]: + raise TuningError("只能回滚到已完成且通过安全门槛的 Trial") + checkpoint = payload.get("checkpoint", False) + if not isinstance(checkpoint, bool): + raise TuningError("checkpoint 必须是布尔值") + if checkpoint: + path_value = target.get("checkpointPath") + if not path_value: + raise TuningError("目标 Trial 没有可用 checkpoint") + root = self._session_root(session_id) + path = (root / path_value).resolve() + if not path.is_relative_to(root) or not path.is_file(): + raise TuningError("目标 checkpoint 不存在或路径非法") + + superseded = [] + for proposal in self.storage.list_proposals(session_id): + if proposal["state"] == "pending" and self.storage.decide_proposal( + proposal["id"], "rejected", f"由回滚到 Trial {target_id[:8]} 取代" + ): + superseded.append(proposal["id"]) + self.storage.set_active_base(session_id, target_id) + self.storage.reset_step_gate(session_id) + self.storage.update_session( + session_id, + state="paused", + message=f"已回滚到 Trial {target['number']} / Rung {target['rung']};等待单步或继续", + ) + self.storage.audit( + session_id, + "rollback_selected", + { + "trialId": target_id, + "checkpoint": checkpoint, + "supersededProposalIds": superseded, + }, + ) + with self.condition: + self.condition.notify_all() + return self.detail(session_id) + def pause(self, session_id: str) -> dict: session = self.storage.get_session(session_id) if session["state"] not in RUNNING_STATES | {"awaiting_approval"}: raise TuningError("当前状态不能暂停") + # 以持久化调度门记录暂停意图,而不是只依赖 session.state。当前 Trial + # 的训练/评估会继续完成,下一次 _claim_dispatch 必须等待显式继续或单步。 + self.storage.reset_step_gate(session_id) self.storage.update_session( - session_id, state="paused", message="已暂停后续调度;当前子进程将完成" + session_id, state="paused", message="已暂停后续调度;在途 Trial(如有)将完整结束" ) return self.detail(session_id) def resume(self, session_id: str) -> dict: session = self.storage.get_session(session_id) if session["state"] == "paused": - self.storage.update_session(session_id, state="running", message="继续调参") + self.storage.set_run_policy(session_id, "continuous") + has_pending = any( + proposal["state"] == "pending" + for proposal in self.storage.list_proposals(session_id) + ) + self.storage.update_session( + session_id, + state="awaiting_approval" if has_pending else "running", + message="请审批待处理 Proposal" if has_pending else "连续调参已恢复", + ) with self.condition: self.condition.notify_all() elif session["state"] == "interrupted": + self.storage.set_run_policy(session_id, "continuous") self.storage.update_session(session_id, state="queued", message="从最近完整结果恢复") self._start_worker(session_id, resume=True) else: @@ -665,7 +937,9 @@ class TuningManager: return self.detail(session_id) def cancel(self, session_id: str) -> dict: - self.storage.get_session(session_id) + session = self.storage.get_session(session_id) + if session["state"] not in ACTIVE_SESSION_STATES: + return self.detail(session_id) self.storage.update_session(session_id, state="cancelled", message="正在取消") event = self.cancel_events.get(session_id) if event: @@ -678,12 +952,22 @@ class TuningManager: return self.detail(session_id) def metrics( - self, session_id: str, trial_id: str, tags: list[str] | None, max_points: int + self, + session_id: str, + trial_id: str, + tags: list[str] | None, + max_points: int, + after_step: int | None = None, ) -> dict: trial = self.storage.get_trial(trial_id) if trial["sessionId"] != session_id: raise TuningError("trial 不属于该 session") - return {"trialId": trial_id, "series": self.storage.metrics(trial_id, tags, max_points)} + series = self.storage.metrics(trial_id, tags, max_points, after_step) + next_step = max( + (point["step"] for item in series for point in item["points"]), + default=after_step, + ) + return {"trialId": trial_id, "series": series, "nextStep": next_step} def best_artifact(self, session_id: str) -> Path: session = self.storage.get_session(session_id) diff --git a/training_server/tuning/schema.py b/training_server/tuning/schema.py index 66f4b09c..e7a2c819 100644 --- a/training_server/tuning/schema.py +++ b/training_server/tuning/schema.py @@ -93,6 +93,68 @@ def _cross_validate(config: Mapping[str, Mapping[str, float]]) -> None: raise RewardConfigError("pose.walking_threshold 必须小于 pose.running_threshold") +def _path_spec(path: str) -> tuple[str, str, NumericSpec]: + if path.startswith("weights."): + section, name = "weights", path.removeprefix("weights.") + spec = WEIGHT_SPECS.get(name) + elif path.startswith("params."): + section, name = "params", path.removeprefix("params.") + spec = PARAMETER_SPECS.get(name) + else: + section, name, spec = "", "", None + if spec is None: + raise RewardConfigError(f"未知参数约束:{path}") + return section, name, spec + + +def validate_constraints(value: Any) -> dict[str, dict[str, float | str]]: + """Validate sparse per-session range/fixed safety constraints.""" + root = _mapping(value, "constraints") + if len(root) > len(WEIGHT_SPECS) + len(PARAMETER_SPECS): + raise RewardConfigError("constraints 数量超过白名单参数总数") + result: dict[str, dict[str, float | str]] = {} + for raw_path, raw_constraint in root.items(): + if not isinstance(raw_path, str): + raise RewardConfigError("constraint path 必须是字符串") + _, _, spec = _path_spec(raw_path) + constraint = _mapping(raw_constraint, raw_path) + kind = constraint.get("kind") + if kind == "fixed": + if set(constraint) != {"kind", "value"}: + raise RewardConfigError(f"{raw_path} fixed 约束只能包含 kind/value") + fixed = _number(f"{raw_path}.value", constraint["value"], spec) + result[raw_path] = {"kind": "fixed", "value": fixed} + elif kind == "range": + if set(constraint) != {"kind", "min", "max"}: + raise RewardConfigError(f"{raw_path} range 约束只能包含 kind/min/max") + minimum = _number(f"{raw_path}.min", constraint["min"], spec) + maximum = _number(f"{raw_path}.max", constraint["max"], spec) + if minimum > maximum: + raise RewardConfigError(f"{raw_path} 下限不能大于上限") + result[raw_path] = {"kind": "range", "min": minimum, "max": maximum} + else: + raise RewardConfigError(f"{raw_path}.kind 必须是 range 或 fixed") + return result + + +def validate_configuration_constraints(value: Any, constraints: Any) -> None: + """Ensure a complete reward configuration satisfies every session constraint.""" + config = validate_configuration(value) + checked = validate_constraints(constraints) + for path, constraint in checked.items(): + section, name, _ = _path_spec(path) + current = config[section][name] + if constraint["kind"] == "fixed": + if current != constraint["value"]: + raise RewardConfigError( + f"{path} 已固定为 {constraint['value']},不能设为 {current}" + ) + elif current < constraint["min"] or current > constraint["max"]: + raise RewardConfigError( + f"{path}={current} 超出工程锁定范围 {constraint['min']}–{constraint['max']}" + ) + + def validate_configuration(value: Any) -> dict[str, dict[str, float]]: """Validate a complete configuration and reject missing/unknown fields.""" root = _mapping(value, "rewardConfig") @@ -118,8 +180,10 @@ def validate_configuration(value: Any) -> dict[str, dict[str, float]]: return config -def validate_proposal(value: Any, previous: Any) -> dict[str, dict[str, float]]: - """Validate a sparse Agent patch relative to a complete previous config.""" +def validate_proposal( + value: Any, previous: Any, constraints: Any | None = None +) -> dict[str, dict[str, float]]: + """Validate a sparse Agent patch relative to a complete previous config and guardrails.""" current = validate_configuration(previous) root = _mapping(value, "proposal") if not set(root).issubset({"weights", "params"}): @@ -167,12 +231,16 @@ def validate_proposal(value: Any, previous: Any) -> dict[str, dict[str, float]]: candidate["weights"].update(patch["weights"]) candidate["params"].update(patch["params"]) _cross_validate(candidate) + if constraints is not None: + validate_configuration_constraints(candidate, constraints) return patch -def merge_proposal(previous: Any, proposal: Any) -> dict[str, dict[str, float]]: +def merge_proposal( + previous: Any, proposal: Any, constraints: Any | None = None +) -> dict[str, dict[str, float]]: current = validate_configuration(previous) - patch = validate_proposal(proposal, current) + patch = validate_proposal(proposal, current, constraints) merged = deepcopy(current) merged["weights"].update(patch["weights"]) merged["params"].update(patch["params"]) diff --git a/training_server/tuning/storage.py b/training_server/tuning/storage.py index 7efae5cd..cfba8abd 100644 --- a/training_server/tuning/storage.py +++ b/training_server/tuning/storage.py @@ -12,7 +12,11 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 + + +class StorageConflict(RuntimeError): + """Optimistic-concurrency revision mismatch.""" def now_iso() -> str: @@ -135,11 +139,29 @@ class TuningStorage: id TEXT PRIMARY KEY, name TEXT NOT NULL UNIQUE, session_id TEXT NOT NULL, trial_id TEXT NOT NULL, reward_config_json TEXT NOT NULL, created_at TEXT NOT NULL ); + CREATE TABLE IF NOT EXISTS session_controls( + session_id TEXT PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE, + run_policy TEXT NOT NULL DEFAULT 'continuous' + CHECK(run_policy IN ('continuous','step')), + dispatch_tokens INTEGER NOT NULL DEFAULT 0 CHECK(dispatch_tokens >= 0), + active_base_trial_id TEXT, + revision INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE IF NOT EXISTS session_constraints( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + path TEXT NOT NULL, kind TEXT NOT NULL CHECK(kind IN ('range','fixed')), + min_value REAL, max_value REAL, fixed_value REAL, + updated_at TEXT NOT NULL, + PRIMARY KEY(session_id, path) + ); CREATE INDEX IF NOT EXISTS idx_trials_session ON trials(session_id, number, rung); CREATE INDEX IF NOT EXISTS idx_proposals_session ON proposals(session_id, created_at); CREATE INDEX IF NOT EXISTS idx_metrics_trial_tag ON metric_points(trial_id, tag, step); """ ) + connection.execute( + "INSERT OR IGNORE INTO session_controls(session_id) SELECT id FROM sessions" + ) connection.execute( "INSERT OR IGNORE INTO schema_migrations(version) VALUES (?)", (SCHEMA_VERSION,) ) @@ -170,6 +192,7 @@ class TuningStorage: "VALUES (?, 'queued', ?, ?, ?, ?, ?, '等待基线训练', ?)", (session_id, mode, at, at, _json(config), _json(objective), int(fallback)), ) + connection.execute("INSERT INTO session_controls(session_id) VALUES (?)", (session_id,)) connection.execute( "INSERT INTO audit_events(session_id,event_type,payload_json,created_at) " "VALUES (?,?,?,?)", @@ -212,6 +235,7 @@ class TuningStorage: def update_session(self, session_id: str, **changes: Any) -> bool: columns = { "state": "state", + "mode": "mode", "message": "message", "current_trial_id": "current_trial_id", "best_trial_id": "best_trial_id", @@ -230,6 +254,143 @@ class TuningStorage: ) return cursor.rowcount == 1 + def get_control(self, session_id: str) -> dict: + row = ( + self.connection() + .execute("SELECT * FROM session_controls WHERE session_id=?", (session_id,)) + .fetchone() + ) + if row is None: + raise KeyError(session_id) + constraint_rows = ( + self.connection() + .execute( + "SELECT * FROM session_constraints WHERE session_id=? ORDER BY path", (session_id,) + ) + .fetchall() + ) + constraints = {} + for constraint in constraint_rows: + if constraint["kind"] == "fixed": + value = {"kind": "fixed", "value": constraint["fixed_value"]} + else: + value = { + "kind": "range", + "min": constraint["min_value"], + "max": constraint["max_value"], + } + constraints[constraint["path"]] = value + return { + "runPolicy": row["run_policy"], + "dispatchTokens": row["dispatch_tokens"], + "constraintsRevision": row["revision"], + "constraints": constraints, + "activeBaseTrialId": row["active_base_trial_id"], + } + + def replace_constraints( + self, session_id: str, expected_revision: int, constraints: dict + ) -> dict: + at = now_iso() + with self.transaction() as connection: + row = connection.execute( + "SELECT revision FROM session_controls WHERE session_id=?", (session_id,) + ).fetchone() + if row is None: + raise KeyError(session_id) + if row["revision"] != expected_revision: + raise StorageConflict( + f"参数护栏 revision 已变化(当前 {row['revision']},请求 {expected_revision})" + ) + connection.execute("DELETE FROM session_constraints WHERE session_id=?", (session_id,)) + for path, constraint in constraints.items(): + connection.execute( + "INSERT INTO session_constraints(" + "session_id,path,kind,min_value,max_value,fixed_value,updated_at) " + "VALUES (?,?,?,?,?,?,?)", + ( + session_id, + path, + constraint["kind"], + constraint.get("min"), + constraint.get("max"), + constraint.get("value"), + at, + ), + ) + connection.execute( + "UPDATE session_controls SET revision=revision+1 WHERE session_id=?", + (session_id,), + ) + return self.get_control(session_id) + + def grant_dispatch_token(self, session_id: str) -> dict: + """Atomically grant the sole outstanding one-Trial token.""" + with self.transaction() as connection: + cursor = connection.execute( + "UPDATE session_controls SET run_policy='step',dispatch_tokens=1 " + "WHERE session_id=? AND dispatch_tokens=0", + (session_id,), + ) + if cursor.rowcount != 1: + exists = connection.execute( + "SELECT 1 FROM session_controls WHERE session_id=?", (session_id,) + ).fetchone() + if exists is None: + raise KeyError(session_id) + raise StorageConflict("已有未消费的单步 Trial 令牌") + return self.get_control(session_id) + + def use_dispatch_token(self, session_id: str) -> bool: + """Atomically consume one step token; continuous mode never needs a token.""" + with self.transaction() as connection: + row = connection.execute( + "SELECT run_policy,dispatch_tokens FROM session_controls WHERE session_id=?", + (session_id,), + ).fetchone() + if row is None: + raise KeyError(session_id) + if row["run_policy"] == "continuous": + return True + if row["dispatch_tokens"] <= 0: + return False + connection.execute( + "UPDATE session_controls SET dispatch_tokens=dispatch_tokens-1 WHERE session_id=?", + (session_id,), + ) + return True + + def set_run_policy(self, session_id: str, policy: str) -> dict: + if policy not in {"continuous", "step"}: + raise ValueError(policy) + cursor = self.connection().execute( + "UPDATE session_controls SET run_policy=?," + "dispatch_tokens=CASE WHEN ?='continuous' THEN 0 ELSE dispatch_tokens END " + "WHERE session_id=?", + (policy, policy, session_id), + ) + if cursor.rowcount != 1: + raise KeyError(session_id) + return self.get_control(session_id) + + def reset_step_gate(self, session_id: str) -> dict: + cursor = self.connection().execute( + "UPDATE session_controls SET run_policy='step',dispatch_tokens=0 WHERE session_id=?", + (session_id,), + ) + if cursor.rowcount != 1: + raise KeyError(session_id) + return self.get_control(session_id) + + def set_active_base(self, session_id: str, trial_id: str | None) -> dict: + cursor = self.connection().execute( + "UPDATE session_controls SET active_base_trial_id=? WHERE session_id=?", + (trial_id, session_id), + ) + if cursor.rowcount != 1: + raise KeyError(session_id) + return self.get_control(session_id) + def create_trial( self, session_id: str, @@ -424,13 +585,20 @@ class TuningStorage: ) def metrics( - self, trial_id: str, tags: list[str] | None = None, max_points: int = 1000 + self, + trial_id: str, + tags: list[str] | None = None, + max_points: int = 1000, + after_step: int | None = None, ) -> list[dict]: parameters: list[Any] = [trial_id] clause = "trial_id=?" if tags: clause += f" AND tag IN ({','.join('?' for _ in tags)})" parameters.extend(tags) + if after_step is not None: + clause += " AND step>?" + parameters.append(after_step) rows = ( self.connection() .execute( diff --git a/web_platform/README.md b/web_platform/README.md index 83265dbf..963e2db9 100644 --- a/web_platform/README.md +++ b/web_platform/README.md @@ -121,7 +121,7 @@ npm run training-server -- \ 服务启动时会在终端输出一个随机访问令牌;在界面中填写该令牌后连接。令牌仅保存在当前标签页的 `sessionStorage`。界面默认连接 `http://127.0.0.1:8765`,可选择服务端允许的任务、并行环境数、训练迭代、随机种子、CPU/GPU、GPU 编号和实验记录方式。W&B 默认为本地离线模式,无需登录或 API Key;也可完全禁用,只有明确选择在线模式时才会联网登录。训练期间页面轮询迭代进度与最近日志,可以停止任务;训练成功后点击“导入策略”,生成的 `policy.onnx` 会进入现有 ONNX 加载流程。普通训练还可以选择自调参产生的命名 reward preset,而不会改写仓库默认配置。 -连接服务后点击“打开自调参 Agent 工作台”会打开独立 `tuning.html`。该页面提供自动/逐轮审批模式、目标权重与预算配置、TensorBoard scalar 筛选/平滑/缩放、trial/rung 排行、固定评估指标、Agent 决策时间线、参数 patch 修改审批、暂停/恢复/停止、最佳 preset JSON 和 ONNX 导出。新标签页 URL 不包含 token;同源 opener 会一次性交接凭据,直接打开页面时也可手工输入。DeepSeek key 始终由本地 Python 服务的 `DEEPSEEK_API_KEY` 环境变量读取,浏览器不会接触该 key。 +连接服务后点击“打开自调参 Agent 工作台”会打开独立 `tuning.html`。该页面采用 Cyber-Industrial 三栏控制台:左侧展示 Session/ASHA 晋级树和 Trial 对比选择,中间使用 uPlot 叠加多 Trial 增量收敛曲线(金线标记历史最优)及六维物理评分,右侧以可折叠因果时间线展示 rationale、expected impact、置信度和评估结果。工具栏支持自动/逐轮审批切换、一次一 Trial 的调度令牌、服务端参数范围/固定值护栏、回滚历史最优或复现任意安全 Trial;Reward Merge Patch 与回滚差异由按需加载的 Monaco Diff 审查。scalar 以 1 Hz 非重入方式增量轮询,进入固定容量环形缓冲并由 `requestAnimationFrame` 合批后调用 `uPlot.setData`,不会在每次轮询时重建图表。新标签页 URL 不包含 token;同源 opener 会一次性交接凭据,直接打开页面时也可手工输入。DeepSeek key 始终由本地 Python 服务的 `DEEPSEEK_API_KEY` 环境变量读取,浏览器不会接触该 key。 桥接服务只监听本机回环地址,并检查 Host、Origin 和 Bearer Token;仅接受允许列表中的任务和经过范围校验的参数,不执行前端提供的 Shell 命令;一次只运行一个训练进程。默认任务使用仓库内置的 Go2 机器人资产与环境配置,**不会自动把浏览器中临时编辑的 MJCF/URDF 作为训练环境**。自定义浏览器模型训练需要在兼容的外部训练工程中注册 task,并通过服务的 `--trainer-root` 指定该工程。服务配置、接口和安全边界见 [`../training_server/README.md`](../training_server/README.md)。 diff --git a/web_platform/src/training/LocalTrainingClient.test.ts b/web_platform/src/training/LocalTrainingClient.test.ts index ceb3d94a..b2b87da1 100644 --- a/web_platform/src/training/LocalTrainingClient.test.ts +++ b/web_platform/src/training/LocalTrainingClient.test.ts @@ -62,13 +62,14 @@ describe('LocalTrainingClient', () => { ); vi.stubGlobal('fetch', fetchMock); const client = new LocalTrainingClient('http://127.0.0.1:8765', 'deep-secret'); - await client.tuningMetrics('a'.repeat(32), 'b'.repeat(32), ['Evaluation/score'], 500); + await client.tuningMetrics('a'.repeat(32), 'b'.repeat(32), ['Evaluation/score'], 500, 120); await client.decideProposal('a'.repeat(32), 'c'.repeat(32), 'approve', { feedback: 'ok', patch: { weights: { pose: 1.1 }, params: {} }, }); expect(fetchMock.mock.calls[0][0]).toContain('/api/tuning/sessions/'); expect(fetchMock.mock.calls[0][0]).toContain('maxPoints=500'); + expect(fetchMock.mock.calls[0][0]).toContain('afterStep=120'); expect(fetchMock.mock.calls[0][0]).not.toContain('deep-secret'); const approval = fetchMock.mock.calls[1][1] as RequestInit; expect(approval.method).toBe('POST'); @@ -119,11 +120,25 @@ describe('LocalTrainingClient', () => { }); await client.tuningSession('a'.repeat(32)); await client.tuningAction('a'.repeat(32), 'pause'); + await client.setTuningMode('a'.repeat(32), 'approval'); + await client.setTuningConstraints('a'.repeat(32), 2, { + 'weights.pose': { kind: 'range', min: 0.5, max: 1.5 }, + }); + await client.stepTuning('a'.repeat(32)); + await client.rollbackTuning('a'.repeat(32), 'b'.repeat(32), true); await client.cancelTuning('a'.repeat(32)); await client.presets(); const policy = await client.downloadBestPolicy('a'.repeat(32)); expect(policy.size).toBe(3); - expect(fetchMock).toHaveBeenCalledTimes(9); + expect(fetchMock).toHaveBeenCalledTimes(13); + const modeRequest = fetchMock.mock.calls.find(([url]) => String(url).endsWith('/mode'))?.[1] as + RequestInit | undefined; + expect(JSON.parse(String(modeRequest?.body))).toEqual({ mode: 'approval' }); + const constraintsRequest = fetchMock.mock.calls.find(([url]) => + String(url).endsWith('/constraints'), + )?.[1] as RequestInit | undefined; + expect(constraintsRequest?.method).toBe('PUT'); + expect(JSON.parse(String(constraintsRequest?.body))).toMatchObject({ revision: 2 }); }); it('拒绝非 HTTP 地址和空访问令牌', () => { diff --git a/web_platform/src/training/LocalTrainingClient.ts b/web_platform/src/training/LocalTrainingClient.ts index 3e4b38d0..336950bd 100644 --- a/web_platform/src/training/LocalTrainingClient.ts +++ b/web_platform/src/training/LocalTrainingClient.ts @@ -1,4 +1,5 @@ import type { + ParameterConstraint, RewardPreset, TuningCapability, TuningCreateRequest, @@ -101,9 +102,41 @@ export class LocalTrainingClient { method: 'POST', }); } + setTuningMode(id: string, mode: TuningSession['mode']): Promise { + return this.json(`/api/tuning/sessions/${encodeURIComponent(id)}/mode`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ mode }), + }); + } cancelTuning(id: string): Promise { return this.json(`/api/tuning/sessions/${encodeURIComponent(id)}`, { method: 'DELETE' }); } + setTuningConstraints( + id: string, + revision: number, + constraints: Record, + ): Promise { + return this.json(`/api/tuning/sessions/${encodeURIComponent(id)}/constraints`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ revision, constraints }), + }); + } + stepTuning(id: string): Promise { + return this.json(`/api/tuning/sessions/${encodeURIComponent(id)}/step`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ count: 1 }), + }); + } + rollbackTuning(id: string, trialId: string, checkpoint = false): Promise { + return this.json(`/api/tuning/sessions/${encodeURIComponent(id)}/rollback`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ trialId, checkpoint }), + }); + } decideProposal( sessionId: string, proposalId: string, @@ -127,11 +160,16 @@ export class LocalTrainingClient { trialId: string, tags: string[] = [], maxPoints = 1000, + afterStep?: number, + signal?: AbortSignal, ): Promise { const query = new URLSearchParams({ maxPoints: String(maxPoints) }); if (tags.length) query.set('tags', tags.join(',')); + if (afterStep !== undefined && Number.isFinite(afterStep)) + query.set('afterStep', String(afterStep)); return this.json( `/api/tuning/sessions/${encodeURIComponent(sessionId)}/trials/${encodeURIComponent(trialId)}/metrics?${query}`, + { signal }, ); } presets(): Promise { diff --git a/web_platform/src/training/LocalTrainingPanel.test.tsx b/web_platform/src/training/LocalTrainingPanel.test.tsx index 3c497a2f..a13644ee 100644 --- a/web_platform/src/training/LocalTrainingPanel.test.tsx +++ b/web_platform/src/training/LocalTrainingPanel.test.tsx @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { LocalTrainingPanel } from './LocalTrainingPanel'; beforeEach(() => { + vi.restoreAllMocks(); localStorage.clear(); sessionStorage.clear(); vi.unstubAllGlobals(); @@ -101,4 +102,60 @@ describe('LocalTrainingPanel', () => { expect(await screen.findByRole('button', { name: '发起本地训练' })).toBeInTheDocument(); expect(sessionStorage.getItem('mujoco-local-training-token')).toBe('new-secret-token'); }); + + it('接收调参窗口传回的策略文件,无需主工作台重复持有令牌', async () => { + const onPolicyReady = vi.fn(); + const reply = vi.spyOn(window, 'postMessage').mockImplementation(() => undefined); + render(); + const policy = new File([new Uint8Array([1, 2, 3])], 'best-policy.onnx', { + type: 'application/octet-stream', + }); + + window.dispatchEvent( + new MessageEvent('message', { + origin: window.location.origin, + source: window, + data: { + type: 'mujoco-tuning-import-policy', + sessionId: 'a'.repeat(32), + policy, + }, + }), + ); + + await waitFor(() => expect(onPolicyReady).toHaveBeenCalledWith(policy)); + expect(reply).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'mujoco-tuning-import-policy-result', + ok: true, + }), + window.location.origin, + ); + }); + + it('旧调参消息缺少主工作台令牌时显示错误而不是抛出未处理异常', async () => { + const reply = vi.spyOn(window, 'postMessage').mockImplementation(() => undefined); + render(); + + window.dispatchEvent( + new MessageEvent('message', { + origin: window.location.origin, + source: window, + data: { + type: 'mujoco-tuning-import-policy', + sessionId: 'a'.repeat(32), + }, + }), + ); + + expect(await screen.findByRole('alert')).toHaveTextContent('请输入训练服务访问令牌'); + expect(reply).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'mujoco-tuning-import-policy-result', + ok: false, + error: '请输入训练服务访问令牌', + }), + window.location.origin, + ); + }); }); diff --git a/web_platform/src/training/LocalTrainingPanel.tsx b/web_platform/src/training/LocalTrainingPanel.tsx index 653a489d..f4f7d3c9 100644 --- a/web_platform/src/training/LocalTrainingPanel.tsx +++ b/web_platform/src/training/LocalTrainingPanel.tsx @@ -129,18 +129,44 @@ export function LocalTrainingPanel({ onPolicyReady }: { onPolicyReady(file: File typeof event.data !== 'object' ) return; - const data = event.data as { type?: string; sessionId?: string }; + const data = event.data as { type?: string; sessionId?: string; policy?: unknown }; + const source = event.source as Window; if (data.type === 'mujoco-tuning-ready') { - (event.source as Window).postMessage( - { type: 'mujoco-tuning-credentials', endpoint, token }, - event.origin, - ); + source.postMessage({ type: 'mujoco-tuning-credentials', endpoint, token }, event.origin); } if (data.type === 'mujoco-tuning-import-policy' && data.sessionId) { - void new LocalTrainingClient(endpoint, token) - .downloadBestPolicy(data.sessionId) - .then(onPolicyReady) - .catch((value: unknown) => setError(errorText(value))); + const reply = (ok: boolean, message?: string) => { + try { + source.postMessage( + { + type: 'mujoco-tuning-import-policy-result', + sessionId: data.sessionId, + ok, + error: message, + }, + event.origin, + ); + } catch { + /* 调参窗口可能已关闭;不影响主工作台继续导入 */ + } + }; + void (async () => { + try { + const policy = + data.policy === undefined + ? await new LocalTrainingClient(endpoint, token).downloadBestPolicy(data.sessionId!) + : data.policy; + if (!(policy instanceof File) || !/\.onnx$/i.test(policy.name)) + throw new Error('调参工作台返回的 ONNX 策略无效'); + if (policy.size > 64 * 1024 * 1024) throw new Error('ONNX 策略不能超过 64 MiB'); + onPolicyReady(policy); + reply(true); + } catch (value) { + const message = errorText(value); + setError(message); + reply(false, message); + } + })(); } }; window.addEventListener('message', receive); diff --git a/web_platform/src/training/types.ts b/web_platform/src/training/types.ts index d82dbd0e..15247b61 100644 --- a/web_platform/src/training/types.ts +++ b/web_platform/src/training/types.ts @@ -43,6 +43,7 @@ export interface TrainingJob { } export type TuningMode = 'automatic' | 'approval'; +export type TuningRunPolicy = 'continuous' | 'step'; export type TuningSessionState = | 'queued' | 'running' @@ -94,11 +95,14 @@ export interface TuningCreateRequest { fallbackEnabled: boolean; } +export type TuningTrialState = + 'queued' | 'training' | 'evaluating' | 'completed' | 'interrupted' | 'failed' | 'cancelled'; + export interface TuningTrial { id: string; sessionId: string; number: number; - state: string; + state: TuningTrialState; rung: number; targetIterations: number; rewardConfig: RewardConfiguration; @@ -138,6 +142,18 @@ export interface TuningAuditEvent { createdAt: string; } +export type ParameterConstraint = + { kind: 'range'; min: number; max: number } | { kind: 'fixed'; value: number }; + +export interface TuningControlState { + runPolicy: TuningRunPolicy; + dispatchTokens: number; + constraintsRevision: number; + constraints: Record; + activeBaseTrialId?: string; + effectiveAfterCurrent: boolean; +} + export interface TuningSession { id: string; state: TuningSessionState; @@ -154,6 +170,8 @@ export interface TuningSession { trials: TuningTrial[]; proposals: TuningProposal[]; audit: TuningAuditEvent[]; + /** 旧服务响应可能缺失;前端会回退到 continuous + 空约束。 */ + control?: TuningControlState; } export interface ScalarPoint { @@ -170,6 +188,8 @@ export interface ScalarSeries { export interface TuningMetricsResponse { trialId: string; series: ScalarSeries[]; + /** 本响应中最大的 step;用于下一次增量请求。 */ + nextStep?: number; } export interface RewardPreset { diff --git a/web_platform/src/tuning/AgentDecisionTimeline.test.tsx b/web_platform/src/tuning/AgentDecisionTimeline.test.tsx new file mode 100644 index 00000000..b0d13262 --- /dev/null +++ b/web_platform/src/tuning/AgentDecisionTimeline.test.tsx @@ -0,0 +1,141 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { TuningSession } from '../training/types'; +import { AgentDecisionTimeline } from './AgentDecisionTimeline'; +import { resetTuningStore, useTuningStore } from './tuningStore'; + +const objectives = { + velocity_tracking: 0.35, + action_smoothness: 0.2, + posture_stability: 0.15, + fall_avoidance: 0.15, + foot_slip: 0.1, + energy: 0.05, +}; + +function fixture(): TuningSession { + const baseConfig = { weights: { pose: 1 }, params: {} }; + const resultConfig = { weights: { pose: 1.1 }, params: {} }; + return { + id: 'a'.repeat(32), + state: 'paused', + mode: 'approval', + createdAt: '2026-01-01T00:00:00Z', + updatedAt: '2026-01-01T00:02:00Z', + config: { + taskId: 'Unitree-Go2-Flat', + mode: 'approval', + runName: 'timeline', + numEnvs: 16, + seed: 42, + gpuIds: [0], + trialCount: 2, + initialIterations: 300, + middleIterations: 900, + finalIterations: 2000, + evalNumEnvs: 8, + evalSteps: 10, + objectiveWeights: objectives, + fallbackEnabled: false, + rungs: [300, 900, 2000], + promote: [2, 2, 2], + }, + objectiveWeights: objectives, + message: 'paused', + bestTrialId: '2'.repeat(32), + consecutiveNoImprove: 0, + fallbackEnabled: false, + trials: [ + { + id: '1'.repeat(32), + sessionId: 'a'.repeat(32), + number: 0, + state: 'completed', + rung: 0, + targetIterations: 300, + rewardConfig: baseConfig, + score: 0, + eligible: true, + createdAt: '2026-01-01T00:00:00Z', + message: 'done', + }, + { + id: '2'.repeat(32), + sessionId: 'a'.repeat(32), + number: 1, + state: 'completed', + rung: 0, + targetIterations: 300, + rewardConfig: resultConfig, + proposalId: '3'.repeat(32), + score: 0.12, + eligible: true, + evaluation: { + metrics: { linear_velocity_rmse: 0.2 }, + score: { score: 0.12, eligible: true, components: { velocity_tracking: 0.2 } }, + }, + createdAt: '2026-01-01T00:01:00Z', + message: 'done', + }, + ], + proposals: [ + { + id: '3'.repeat(32), + sessionId: 'a'.repeat(32), + baseTrialId: '1'.repeat(32), + state: 'approved', + source: 'agent', + patch: { weights: { pose: 1.1 }, params: {} }, + rationale: '提高姿态奖励以降低躯干倾角。', + expectedImpact: { posture_stability: '姿态误差预计下降 8%' }, + confidence: 0.82, + createdAt: '2026-01-01T00:00:30Z', + }, + ], + audit: [], + control: { + runPolicy: 'step', + dispatchTokens: 0, + constraintsRevision: 0, + constraints: {}, + effectiveAfterCurrent: false, + }, + }; +} + +beforeEach(() => { + resetTuningStore(); + vi.unstubAllGlobals(); + useTuningStore.getState().setConnection('http://127.0.0.1:8765', 'secret'); + useTuningStore.getState().applySessionSnapshot(fixture()); +}); + +describe('AgentDecisionTimeline', () => { + it('展示 rationale、预期影响、置信度与 Proposal→Trial 因果结果', () => { + render(); + expect(screen.getByText('提高姿态奖励以降低躯干倾角。')).toBeInTheDocument(); + expect(screen.getByText('姿态误差预计下降 8%')).toBeInTheDocument(); + expect(screen.getByText('82%')).toBeInTheDocument(); + expect(screen.getByText('+0.1200')).toBeInTheDocument(); + expect(screen.getByText('→ 1.1000')).toBeInTheDocument(); + }); + + it('在安全边界一键把关联 Trial 设为复现基准', async () => { + const response = fixture(); + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify(response), { + status: 202, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + render(); + fireEvent.click(screen.getByRole('button', { name: '一键复现该轮' })); + await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1)); + expect(String(fetchMock.mock.calls[0][0])).toMatch(/\/rollback$/); + expect(JSON.parse(String(fetchMock.mock.calls[0][1]?.body))).toEqual({ + trialId: '2'.repeat(32), + checkpoint: false, + }); + }); +}); diff --git a/web_platform/src/tuning/AgentDecisionTimeline.tsx b/web_platform/src/tuning/AgentDecisionTimeline.tsx new file mode 100644 index 00000000..be7e56d3 --- /dev/null +++ b/web_platform/src/tuning/AgentDecisionTimeline.tsx @@ -0,0 +1,478 @@ +import { lazy, Suspense, useMemo, useState } from 'react'; +import { + Bot, + BrainCircuit, + Check, + ChevronDown, + ChevronRight, + CircleDot, + CopyCheck, + GitCompareArrows, + RotateCcw, + Sparkles, + Target, + X, +} from 'lucide-react'; +import { useShallow } from 'zustand/react/shallow'; +import { Badge, Button, Dialog } from '../components/ui'; +import type { + RewardConfiguration, + TuningProposal, + TuningSessionState, + TuningTrial, +} from '../training/types'; +import { + formatMetric, + mergeRewardPatch, + OBJECTIVE_META, + PARAMETER_BY_PATH, + rewardConfigurationDiff, +} from './domain'; +import { useTuningStore } from './tuningStore'; + +const RewardConfigDiffEditor = lazy(() => + import('./RewardConfigDiffEditor').then((module) => ({ default: module.RewardConfigDiffEditor })), +); + +interface DecisionView { + proposal: TuningProposal; + base?: TuningTrial; + result?: TuningTrial; +} + +function trialForProposal( + trials: readonly TuningTrial[], + proposalId: string, +): TuningTrial | undefined { + return trials + .filter((trial) => trial.proposalId === proposalId) + .sort((left, right) => right.rung - left.rung)[0]; +} + +function parseEditedConfiguration( + text: string, + base: RewardConfiguration, +): TuningProposal['patch'] { + const value = JSON.parse(text) as Partial; + if (!value || typeof value !== 'object' || !value.weights || !value.params) + throw new Error('编辑结果必须是包含 weights 与 params 的完整 Reward Configuration'); + const patch: TuningProposal['patch'] = { weights: {}, params: {} }; + for (const section of ['weights', 'params'] as const) { + const editedSection = value[section]!; + if (Object.keys(editedSection).some((key) => !(key in base[section]))) + throw new Error(`${section} 包含服务端白名单之外的参数`); + if (Object.keys(base[section]).some((key) => !(key in editedSection))) + throw new Error(`${section} 不能删除参数`); + for (const [key, previous] of Object.entries(base[section])) { + const next = editedSection[key]; + if (typeof next !== 'number' || !Number.isFinite(next)) + throw new Error(`${section}.${key} 必须是有限数值`); + if (next !== previous) patch[section][key] = next; + } + } + const count = Object.keys(patch.weights).length + Object.keys(patch.params).length; + if (!count) throw new Error('修改结果与基准完全相同'); + if (count > 4) throw new Error('每轮最多修改 4 个标量'); + return patch; +} + +function ProposalReviewDialog({ + view, + close, +}: { + view: DecisionView & { base: TuningTrial }; + close(): void; +}) { + const baseConfig = view.base.rewardConfig; + const candidate = mergeRewardPatch(baseConfig, view.proposal.patch); + const [edited, setEdited] = useState(() => JSON.stringify(candidate, null, 2)); + const [feedback, setFeedback] = useState(''); + const [problem, setProblem] = useState(); + const busy = useTuningStore((state) => + state.busyOperations.includes(`proposal:${view.proposal.id}`), + ); + const decide = async (action: 'approve' | 'reject') => { + try { + setProblem(undefined); + const patch = action === 'approve' ? parseEditedConfiguration(edited, baseConfig) : undefined; + await useTuningStore.getState().decideProposal(view.proposal.id, action, { + feedback: feedback.trim() || undefined, + patch, + }); + if (!useTuningStore.getState().error) close(); + } catch (value) { + setProblem(value instanceof Error ? value.message : String(value)); + } + }; + + return ( + + +
+ + +
+ + } + > +
+

+ Monaco 左侧为不可变基准,右侧为可编辑候选。提交时自动转换成稀疏 Merge + Patch;服务端会再次执行白名单、单轮变化率与参数护栏校验。 +

+ 置信度 {Math.round(view.proposal.confidence * 100)}% +
+ {problem && ( +

+ {problem} +

+ )} +
+ + 正在按需加载 Monaco JSON Diff… +
+ } + > + + + +
+ ); +} + +function PatchRows({ view }: { view: DecisionView }) { + if (!view.base) + return ( +
+        {JSON.stringify(view.proposal.patch, null, 2)}
+      
+ ); + const candidate = mergeRewardPatch(view.base.rewardConfig, view.proposal.patch); + const changes = rewardConfigurationDiff(view.base.rewardConfig, candidate); + return ( +
+ {changes.map((change) => { + const definition = PARAMETER_BY_PATH.get(change.path); + return ( +
+
+

+ {definition?.label ?? change.path} +

+ {change.path} +
+ + {formatMetric(change.before)} + + + → {formatMetric(change.after)} + +
+ ); + })} +
+ ); +} + +function ExpectedImpact({ proposal }: { proposal: TuningProposal }) { + const values = Object.entries(proposal.expectedImpact ?? {}); + if (!values.length) + return

Agent 未提供维度预测。

; + return ( +
+ {values.map(([key, value]) => { + const label = OBJECTIVE_META.find((item) => item.key === key)?.label ?? key; + return ( +
+

{label}

+

{String(value)}

+
+ ); + })} +
+ ); +} + +function ResultSummary({ result, base }: { result?: TuningTrial; base?: TuningTrial }) { + if (!result) + return

尚未生成关联 Trial。

; + const delta = + result.score !== undefined && + result.score !== null && + base?.score !== undefined && + base.score !== null + ? result.score - base.score + : undefined; + return ( +
+
+

结果

+

+ T{result.number} · R{result.rung} +

+
+
+

Score

+

+ {formatMetric(result.score, 5)} +

+
+
+

因果结果

+

+ {delta === undefined + ? result.eligible + ? '安全门通过' + : '安全门拒绝' + : `${delta >= 0 ? '+' : ''}${delta.toFixed(4)}`} +

+
+
+ ); +} + +function canReplay(state: TuningSessionState | undefined): boolean { + return state === 'paused' || state === 'awaiting_approval'; +} + +export function AgentDecisionTimeline() { + const [proposalIds, trialIds, entitiesRevision, sessionState] = useTuningStore( + useShallow( + (state) => + [state.proposalIds, state.trialIds, state.entitiesRevision, state.sessionState] as const, + ), + ); + const views = useMemo(() => { + void entitiesRevision; + const state = useTuningStore.getState(); + const trials = trialIds.map((id) => state.trialsById[id]).filter(Boolean); + return proposalIds + .map((id) => state.proposalsById[id]) + .filter(Boolean) + .map((proposal) => ({ + proposal, + base: proposal.baseTrialId ? state.trialsById[proposal.baseTrialId] : undefined, + result: trialForProposal(trials, proposal.id), + })) + .reverse(); + }, [entitiesRevision, proposalIds, trialIds]); + const [expandedId, setExpandedId] = useState(); + const [reviewing, setReviewing] = useState(); + const effectiveExpandedId = expandedId === undefined ? views[0]?.proposal.id : expandedId; + + return ( + <> +
+
+
+

+ Decision Timeline +

+

Proposal → 参数变化 → 固定评估

+
+ {views.length} 轮决策 +
+
+ {views.length === 0 ? ( +
+
+ + 基线完成后,Agent 决策会出现在这里 +
+
+ ) : ( +
    + {views.map((view, index) => { + const open = effectiveExpandedId === view.proposal.id; + const pending = view.proposal.state === 'pending'; + const contentId = `proposal-${view.proposal.id}`; + return ( +
  1. +
    + {index < views.length - 1 && ( + + )} + + + +
    +
    + + {open && ( +
    +
    +

    + Agent rationale +

    +

    + {view.proposal.rationale} +

    +
    +
    +

    + 参数因果变更 +

    + +
    +
    +

    + Expected impact +

    + +
    +
    +
    + 模型置信度 + + {Math.round(view.proposal.confidence * 100)}% + +
    +
    +
    +
    +
    + +
    + {pending && ( + + )} + {view.result?.state === 'completed' && ( + + )} +
    +
    + )} +
    +
  2. + ); + })} +
+ )} +
+
+ {reviewing?.base && ( + setReviewing(undefined)} + /> + )} + + ); +} diff --git a/web_platform/src/tuning/MetricsComparisonBoard.test.tsx b/web_platform/src/tuning/MetricsComparisonBoard.test.tsx new file mode 100644 index 00000000..4b580e0a --- /dev/null +++ b/web_platform/src/tuning/MetricsComparisonBoard.test.tsx @@ -0,0 +1,116 @@ +import { act, render, screen, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { MetricsComparisonBoard } from './MetricsComparisonBoard'; +import { resetTuningStore, useTuningStore } from './tuningStore'; + +const plotSpies = vi.hoisted(() => ({ + constructors: vi.fn(), + setData: vi.fn(), + destroy: vi.fn(), +})); + +vi.mock('uplot', () => { + class FakeUPlot { + data: unknown[]; + over = document.createElement('div'); + scales = { x: { min: 0, max: 10 }, y: { min: 0, max: 10 } }; + series: Array<{ scale?: string }>; + ctx = { + save: () => undefined, + restore: () => undefined, + beginPath: () => undefined, + arc: () => undefined, + fill: () => undefined, + stroke: () => undefined, + }; + + constructor(options: { series: Array<{ scale?: string }> }, data: unknown[]) { + this.data = data; + this.series = options.series; + plotSpies.constructors(); + } + + setData(data: unknown[]): void { + this.data = data; + plotSpies.setData(); + } + + setSize(): void {} + setScale(): void {} + valToPos(value: number): number { + return value; + } + destroy(): void { + plotSpies.destroy(); + } + } + return { default: FakeUPlot }; +}); + +beforeEach(() => { + resetTuningStore(); + plotSpies.constructors.mockClear(); + plotSpies.setData.mockClear(); + plotSpies.destroy.mockClear(); + vi.stubGlobal('requestAnimationFrame', (callback: FrameRequestCallback) => { + callback(0); + return 1; + }); + vi.stubGlobal('cancelAnimationFrame', () => undefined); + const first = { + id: 'a', + sessionId: 's', + number: 0, + state: 'completed' as const, + rung: 0, + targetIterations: 300, + rewardConfig: { weights: {}, params: {} }, + score: 0, + eligible: true, + createdAt: '', + message: '', + }; + const best = { ...first, id: 'b', number: 1, score: 0.2 }; + useTuningStore.setState({ + visibleTrialIds: ['a', 'b'], + bestTrialId: 'b', + selectedTrialId: 'a', + trialsById: { a: first, b: best }, + trialIds: ['a', 'b'], + metricTag: 'Train/reward', + knownMetricTags: ['Train/reward'], + }); + useTuningStore.getState().enqueueMetricBatch({ + trialId: 'a', + series: [{ tag: 'Train/reward', points: [{ step: 1, wallTime: 1, value: 1 }] }], + }); + useTuningStore.getState().enqueueMetricBatch({ + trialId: 'b', + series: [{ tag: 'Train/reward', points: [{ step: 1, wallTime: 1, value: 2 }] }], + }); + useTuningStore.getState().flushMetricBatches(); +}); + +describe('MetricsComparisonBoard', () => { + it('scalar 更新只调用 setData,不重建 uPlot,并在卸载时销毁', async () => { + const view = render(); + expect(screen.getByRole('img', { name: /2 个 Trial/ })).toBeInTheDocument(); + await waitFor(() => expect(plotSpies.constructors).toHaveBeenCalledTimes(1)); + const previousUpdates = plotSpies.setData.mock.calls.length; + + act(() => { + useTuningStore.getState().enqueueMetricBatch({ + trialId: 'a', + series: [{ tag: 'Train/reward', points: [{ step: 2, wallTime: 2, value: 3 }] }], + }); + useTuningStore.getState().flushMetricBatches(); + }); + + await waitFor(() => + expect(plotSpies.setData.mock.calls.length).toBeGreaterThan(previousUpdates), + ); + expect(plotSpies.constructors).toHaveBeenCalledTimes(1); + view.unmount(); + expect(plotSpies.destroy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/web_platform/src/tuning/MetricsComparisonBoard.tsx b/web_platform/src/tuning/MetricsComparisonBoard.tsx new file mode 100644 index 00000000..fa2f086d --- /dev/null +++ b/web_platform/src/tuning/MetricsComparisonBoard.tsx @@ -0,0 +1,495 @@ +import { useEffect, useMemo, useRef } from 'react'; +import { Activity, Crosshair, RotateCcw, TrendingUp, Trophy, ZoomIn, ZoomOut } from 'lucide-react'; +import uPlot from 'uplot'; +import { useShallow } from 'zustand/react/shallow'; +import { Badge, Button, Select } from '../components/ui'; +import type { ScalarPoint, TuningTrial } from '../training/types'; +import { formatMetric, OBJECTIVE_META } from './domain'; +import { useTuningStore } from './tuningStore'; + +const LINE_COLORS = ['#60a5fa', '#a78bfa', '#f472b6', '#2dd4bf', '#fb7185', '#94a3b8']; +const BEST_COLOR = '#f3bd5c'; +const SELECTED_COLOR = '#38d39f'; +const PREFERRED_TAGS = [ + 'Train/mean_reward', + 'Evaluation/linear_velocity_rmse', + 'Episode_Reward/track_linear_velocity', +]; + +interface TrialCurve { + trialId: string; + label: string; + points: ScalarPoint[]; + color: string; + best: boolean; + selected: boolean; +} + +function smooth(values: (number | null)[], factor: number): (number | null)[] { + if (factor <= 0) return values; + let previous: number | undefined; + return values.map((value) => { + if (value === null) return null; + previous = previous === undefined ? value : factor * previous + (1 - factor) * value; + return previous; + }); +} + +function alignCurves(curves: readonly TrialCurve[], smoothing: number): uPlot.AlignedData { + const steps = Array.from( + new Set(curves.flatMap((curve) => curve.points.map((point) => point.step))), + ).sort((left, right) => left - right); + const data: uPlot.AlignedData = [steps]; + for (const curve of curves) { + const byStep = new Map(curve.points.map((point) => [point.step, point.value])); + data.push( + smooth( + steps.map((step) => byStep.get(step) ?? null), + smoothing, + ), + ); + } + return data; +} + +function bestNodePlugin(bestSeriesIndex: number): uPlot.Plugin { + return { + hooks: { + draw: [ + (chart) => { + if (bestSeriesIndex < 1) return; + const xValues = chart.data[0]; + const yValues = chart.data[bestSeriesIndex]; + let index = yValues.length - 1; + while (index >= 0 && yValues[index] === null) index -= 1; + if (index < 0) return; + const xValue = xValues[index]; + const yValue = yValues[index]; + if (xValue === undefined || yValue === null || yValue === undefined) return; + const x = chart.valToPos(xValue, 'x', true); + const y = chart.valToPos(yValue, chart.series[bestSeriesIndex].scale ?? 'y', true); + const context = chart.ctx; + context.save(); + context.shadowColor = BEST_COLOR; + context.shadowBlur = 10 * devicePixelRatio; + context.fillStyle = BEST_COLOR; + context.strokeStyle = '#09111e'; + context.lineWidth = 2 * devicePixelRatio; + context.beginPath(); + context.arc(x, y, 5 * devicePixelRatio, 0, Math.PI * 2); + context.fill(); + context.stroke(); + context.restore(); + }, + ], + }, + }; +} + +function MultiTrialPlot({ + curves, + smoothing, + tag, +}: { + curves: TrialCurve[]; + smoothing: number; + tag: string; +}) { + const host = useRef(null); + const chartRef = useRef(null); + const data = useMemo(() => alignCurves(curves, smoothing), [curves, smoothing]); + const dataRef = useRef(data); + const manualZoom = useRef(false); + const schema = curves + .map((curve) => `${curve.trialId}:${curve.label}:${curve.color}:${curve.best}`) + .join('|'); + + useEffect(() => { + const element = host.current; + if (!element || curves.length === 0) return; + let resizeFrame = 0; + const width = Math.max(320, Math.floor(element.getBoundingClientRect().width)); + const bestIndex = curves.findIndex((curve) => curve.best); + const chart = new uPlot( + { + width, + height: 338, + padding: [12, 12, 0, 0], + legend: { show: true, live: true }, + cursor: { + drag: { x: true, y: false, setScale: true, dist: 8 }, + focus: { prox: 24 }, + points: { size: 7 }, + }, + focus: { alpha: 0.22 }, + scales: { x: { time: false } }, + axes: [ + { + label: 'Iteration', + stroke: '#8fa0b5', + grid: { stroke: '#213044', width: 1 }, + ticks: { stroke: '#40566f' }, + }, + { + stroke: '#8fa0b5', + grid: { stroke: '#213044', width: 1 }, + ticks: { stroke: '#40566f' }, + size: 58, + }, + ], + series: [ + { label: 'Iteration' }, + ...curves.map((curve) => ({ + label: curve.label, + stroke: curve.color, + width: curve.best ? 3 : curve.selected ? 2.5 : 1.5, + alpha: curve.best || curve.selected ? 1 : 0.6, + spanGaps: true, + points: { show: false }, + })), + ], + plugins: [bestNodePlugin(bestIndex < 0 ? -1 : bestIndex + 1)], + }, + // schema(曲线数量)变化时必须使用本次 render 的数据,不能使用要到下一 + // 个 effect 才更新的 ref,否则 uPlot 会短暂收到错误的列数。 + data, + element, + ); + chartRef.current = chart; + + const markManualZoom = () => { + manualZoom.current = true; + }; + chart.over.addEventListener('mousedown', markManualZoom); + const wheelZoom = (event: WheelEvent) => { + if (!event.deltaY) return; + event.preventDefault(); + manualZoom.current = true; + const bounds = chart.over.getBoundingClientRect(); + if (!bounds.width) return; + const scale = chart.scales.x; + if (typeof scale.min !== 'number' || typeof scale.max !== 'number') return; + const ratio = Math.min(1, Math.max(0, (event.clientX - bounds.left) / bounds.width)); + const anchor = scale.min + (scale.max - scale.min) * ratio; + const factor = Math.min(1.8, Math.max(0.55, Math.exp(event.deltaY * 0.0015))); + chart.setScale('x', { + min: anchor - (anchor - scale.min) * factor, + max: anchor + (scale.max - anchor) * factor, + }); + }; + chart.over.addEventListener('wheel', wheelZoom, { passive: false }); + + let lastWidth = width; + const observer = new ResizeObserver(() => { + cancelAnimationFrame(resizeFrame); + resizeFrame = requestAnimationFrame(() => { + const nextWidth = Math.max(320, Math.floor(element.getBoundingClientRect().width)); + if (nextWidth !== lastWidth) { + lastWidth = nextWidth; + chart.setSize({ width: nextWidth, height: 338 }); + } + }); + }); + observer.observe(element); + return () => { + observer.disconnect(); + cancelAnimationFrame(resizeFrame); + chart.over.removeEventListener('mousedown', markManualZoom); + chart.over.removeEventListener('wheel', wheelZoom); + chartRef.current = null; + chart.destroy(); + }; + // schema 仅在曲线身份/样式变化时改变;数据更新由下方 setData effect 处理。 + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [schema]); + + useEffect(() => { + dataRef.current = data; + const chart = chartRef.current; + if (!chart) return; + const frame = requestAnimationFrame(() => chart.setData(data, !manualZoom.current)); + return () => cancelAnimationFrame(frame); + }, [data]); + + const zoom = (factor: number) => { + const chart = chartRef.current; + if (!chart) return; + manualZoom.current = true; + const scale = chart.scales.x; + if (typeof scale.min !== 'number' || typeof scale.max !== 'number') return; + const center = (scale.min + scale.max) / 2; + const radius = ((scale.max - scale.min) * factor) / 2 || 1; + chart.setScale('x', { min: center - radius, max: center + radius }); + }; + const reset = () => { + const chart = chartRef.current; + if (!chart) return; + manualZoom.current = false; + chart.setData(dataRef.current, true); + }; + + return ( +
+
+
+

+ {tag} +

+

拖拽框选 · 滚轮缩放 · hover 对齐

+
+
+ + + +
+
+
+
+ ); +} + +function ScoreBreakdown({ current, best }: { current?: TuningTrial; best?: TuningTrial }) { + const currentComponents = current?.evaluation?.score?.components ?? {}; + const bestComponents = best?.evaluation?.score?.components ?? {}; + return ( +
+
+
+

Score Breakdown

+

相对基线改善 · 中线为 0

+
+ {best && ( + + T{best.number} 最优 + + )} +
+
+ {OBJECTIVE_META.map(({ key, label }) => { + const currentValue = currentComponents[key] ?? 0; + const bestValue = bestComponents[key] ?? 0; + const currentWidth = Math.min(50, Math.abs(currentValue) * 50); + const bestPosition = 50 + Math.max(-1, Math.min(1, bestValue)) * 50; + return ( +
+
+ {label} + + {currentValue >= 0 ? '+' : ''} + {formatMetric(currentValue, 3)} + +
+
+ + + +
+
+ ); + })} +
+
+ ); +} + +function MetricKpis({ current, best }: { current?: TuningTrial; best?: TuningTrial }) { + const currentMetrics = current?.evaluation?.metrics ?? {}; + const bestMetrics = best?.evaluation?.metrics ?? {}; + return ( +
+ {OBJECTIVE_META.map(({ key, shortLabel, metric }) => { + const currentValue = currentMetrics[metric]; + const bestValue = bestMetrics[metric]; + const worse = + currentValue !== undefined && bestValue !== undefined && currentValue > bestValue; + return ( +
+

+ {shortLabel} +

+

+ {formatMetric(currentValue)} +

+

+ best {formatMetric(bestValue)} +

+
+ ); + })} +
+ ); +} + +export function MetricsComparisonBoard() { + const [visibleTrialIds, bestTrialId, selectedTrialId, metricTag, smoothing] = useTuningStore( + useShallow( + (state) => + [ + state.visibleTrialIds, + state.bestTrialId, + state.selectedTrialId, + state.metricTag, + state.smoothing, + ] as const, + ), + ); + const [knownMetricTags, metricsRevision] = useTuningStore( + useShallow((state) => [state.knownMetricTags, state.metricsRevision] as const), + ); + const current = useTuningStore((state) => + selectedTrialId ? state.trialsById[selectedTrialId] : undefined, + ); + const best = useTuningStore((state) => (bestTrialId ? state.trialsById[bestTrialId] : undefined)); + const effectiveTag = + metricTag || + PREFERRED_TAGS.find((tag) => knownMetricTags.includes(tag)) || + knownMetricTags[0] || + ''; + + useEffect(() => { + if (!metricTag && effectiveTag) useTuningStore.getState().setMetricTag(effectiveTag); + }, [effectiveTag, metricTag]); + + const curves = useMemo(() => { + void metricsRevision; + if (!effectiveTag) return []; + const snapshots = useTuningStore.getState().readMetricSeries(visibleTrialIds, effectiveTag); + const ordered = [...snapshots].sort((left, right) => { + if (left.trialId === bestTrialId) return 1; + if (right.trialId === bestTrialId) return -1; + return visibleTrialIds.indexOf(left.trialId) - visibleTrialIds.indexOf(right.trialId); + }); + return ordered.map((series, index) => { + const trial = useTuningStore.getState().trialsById[series.trialId]; + const isBest = series.trialId === bestTrialId; + const isSelected = series.trialId === selectedTrialId; + return { + trialId: series.trialId, + label: trial + ? `T${trial.number} · R${trial.rung}${isBest ? ' · BEST' : ''}` + : series.trialId.slice(0, 6), + points: series.points, + best: isBest, + selected: isSelected, + color: isBest + ? BEST_COLOR + : isSelected + ? SELECTED_COLOR + : LINE_COLORS[index % LINE_COLORS.length], + } satisfies TrialCurve; + }); + }, [bestTrialId, effectiveTag, metricsRevision, selectedTrialId, visibleTrialIds]); + + const ensureBestVisible = () => { + if (bestTrialId) useTuningStore.getState().setTrialVisible(bestTrialId, true); + }; + + return ( +
+
+
+
+

+ 多 Trial 收敛流 +

+

+ 增量 scalar · 4096 点环形缓冲 · rAF 合批 +

+
+
+ + + {bestTrialId && !visibleTrialIds.includes(bestTrialId) && ( + + )} +
+
+
+ + + + {curves.length && effectiveTag ? ( + + ) : ( +
+
+ + 选择 Trial 后等待增量 scalar 数据 +
+
+ )} + + +
+ ); +} diff --git a/web_platform/src/tuning/RewardConfigDiffEditor.tsx b/web_platform/src/tuning/RewardConfigDiffEditor.tsx new file mode 100644 index 00000000..370bbadc --- /dev/null +++ b/web_platform/src/tuning/RewardConfigDiffEditor.tsx @@ -0,0 +1,61 @@ +import { loader, DiffEditor } from '@monaco-editor/react'; +import * as monaco from 'monaco-editor/editor/editor.api'; +import 'monaco-editor/languages/features/json/register'; +import EditorWorker from 'monaco-editor/editor/editor.worker?worker'; +import JsonWorker from 'monaco-editor/language/json/json.worker?worker'; +import type { RewardConfiguration } from '../training/types'; + +type MonacoGlobal = typeof globalThis & { + MonacoEnvironment?: { getWorker?: (_moduleId: string, label: string) => Worker }; +}; + +(globalThis as MonacoGlobal).MonacoEnvironment = { + getWorker: (_moduleId, label) => (label === 'json' ? new JsonWorker() : new EditorWorker()), +}; +loader.config({ monaco }); + +export function RewardConfigDiffEditor({ + original, + modified, + height = 360, + readOnly = true, + onModifiedChange, +}: { + original: RewardConfiguration; + modified: RewardConfiguration; + height?: number; + readOnly?: boolean; + onModifiedChange?: (value: string) => void; +}) { + return ( + { + const model = editor.getModifiedEditor().getModel(); + if (model && onModifiedChange) { + onModifiedChange(model.getValue()); + model.onDidChangeContent(() => onModifiedChange(model.getValue())); + } + }} + options={{ + readOnly, + originalEditable: false, + renderSideBySide: true, + minimap: { enabled: false }, + fontSize: 11, + lineNumbersMinChars: 3, + folding: true, + automaticLayout: true, + scrollBeyondLastLine: false, + wordWrap: 'on', + padding: { top: 8, bottom: 8 }, + }} + /> + ); +} diff --git a/web_platform/src/tuning/ScalarRingBuffer.test.ts b/web_platform/src/tuning/ScalarRingBuffer.test.ts new file mode 100644 index 00000000..f7f37a83 --- /dev/null +++ b/web_platform/src/tuning/ScalarRingBuffer.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest'; +import { ScalarRingBuffer } from './ScalarRingBuffer'; + +const point = (step: number, value = step) => ({ step, wallTime: step / 10, value }); + +describe('ScalarRingBuffer', () => { + it('固定容量覆盖最旧点并保持按 step 排序', () => { + const buffer = new ScalarRingBuffer(3); + buffer.appendMany([point(2), point(1), point(3), point(4)]); + expect(buffer.size).toBe(3); + expect(buffer.snapshot().map((value) => value.step)).toEqual([1, 3, 4]); + expect(buffer.latestStep).toBe(4); + }); + + it('按 step 去重更新且忽略非有限数据', () => { + const buffer = new ScalarRingBuffer(4); + expect(buffer.append(point(1, 2))).toBe(true); + expect(buffer.append(point(1, 2))).toBe(false); + expect(buffer.append(point(1, 3))).toBe(true); + expect(buffer.append({ step: 2, wallTime: 0, value: Number.NaN })).toBe(false); + expect(buffer.snapshot()).toEqual([{ step: 1, wallTime: 0.1, value: 3 }]); + }); +}); diff --git a/web_platform/src/tuning/ScalarRingBuffer.ts b/web_platform/src/tuning/ScalarRingBuffer.ts new file mode 100644 index 00000000..b8d0ef38 --- /dev/null +++ b/web_platform/src/tuning/ScalarRingBuffer.ts @@ -0,0 +1,96 @@ +import type { ScalarPoint, ScalarSeries } from '../training/types'; + +/** + * 固定容量、按 step 去重的数值环形缓冲。 + * + * 网络轮询通常按 step 递增,但服务恢复时可能重发旧点,因此保留 step -> 物理槽位 + * 索引;覆盖最旧槽位时同步删除索引。React 不订阅实例本身,只订阅 store 中的版本号。 + */ +export class ScalarRingBuffer { + readonly capacity: number; + private readonly steps: Float64Array; + private readonly wallTimes: Float64Array; + private readonly values: Float64Array; + private readonly slotByStep = new Map(); + private head = 0; + private length = 0; + + constructor(capacity = 4096) { + if (!Number.isInteger(capacity) || capacity < 2) + throw new Error('环形缓冲容量必须是大于 1 的整数'); + this.capacity = capacity; + this.steps = new Float64Array(capacity); + this.wallTimes = new Float64Array(capacity); + this.values = new Float64Array(capacity); + } + + get size(): number { + return this.length; + } + + get latestStep(): number | undefined { + if (!this.length) return undefined; + let latest = Number.NEGATIVE_INFINITY; + for (const step of this.slotByStep.keys()) latest = Math.max(latest, step); + return Number.isFinite(latest) ? latest : undefined; + } + + append(point: ScalarPoint): boolean { + if (![point.step, point.wallTime, point.value].every(Number.isFinite)) return false; + const existing = this.slotByStep.get(point.step); + if (existing !== undefined) { + if (this.wallTimes[existing] === point.wallTime && this.values[existing] === point.value) + return false; + this.wallTimes[existing] = point.wallTime; + this.values[existing] = point.value; + return true; + } + + let slot: number; + if (this.length < this.capacity) { + slot = (this.head + this.length) % this.capacity; + this.length += 1; + } else { + slot = this.head; + this.slotByStep.delete(this.steps[slot]); + this.head = (this.head + 1) % this.capacity; + } + this.steps[slot] = point.step; + this.wallTimes[slot] = point.wallTime; + this.values[slot] = point.value; + this.slotByStep.set(point.step, slot); + return true; + } + + appendMany(points: readonly ScalarPoint[]): boolean { + let changed = false; + for (const point of points) changed = this.append(point) || changed; + return changed; + } + + snapshot(): ScalarPoint[] { + const points = Array.from(this.slotByStep, ([step, slot]) => ({ + step, + wallTime: this.wallTimes[slot], + value: this.values[slot], + })); + points.sort((left, right) => left.step - right.step); + return points; + } + + clear(): void { + this.slotByStep.clear(); + this.head = 0; + this.length = 0; + } +} + +export interface BufferedMetricSeries { + trialId: string; + tag: string; + buffer: ScalarRingBuffer; +} + +export function snapshotSeries(series: BufferedMetricSeries): ScalarSeries { + return { tag: series.tag, points: series.buffer.snapshot() }; +} diff --git a/web_platform/src/tuning/TuningApp.test.tsx b/web_platform/src/tuning/TuningApp.test.tsx index 2926b39e..82cf4b8b 100644 --- a/web_platform/src/tuning/TuningApp.test.tsx +++ b/web_platform/src/tuning/TuningApp.test.tsx @@ -35,13 +35,18 @@ describe('TuningApp', () => { ); }); vi.stubGlobal('fetch', fetchMock); - render(); + const { container } = render(); + expect(container.firstElementChild).toHaveClass('h-full', 'overflow-y-auto'); fireEvent.change(screen.getByLabelText('访问令牌(仅当前标签页)'), { target: { value: 'training-secret' }, }); fireEvent.click(screen.getByRole('button', { name: '连接/刷新' })); expect(await screen.findByText(/deepseek-v4-flash/)).toBeInTheDocument(); expect(screen.getByText('新建 Unitree-Go2-Flat 调参 Session')).toBeInTheDocument(); + const trialCount = screen.getByLabelText('调参次数(候选配置数)'); + expect(trialCount).toHaveValue(12); + fireEvent.change(trialCount, { target: { value: '36' } }); + expect(trialCount).toHaveValue(36); await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); for (const call of fetchMock.mock.calls) { expect(String(call[0])).not.toContain('training-secret'); diff --git a/web_platform/src/tuning/TuningApp.tsx b/web_platform/src/tuning/TuningApp.tsx index b00a373a..7379f72f 100644 --- a/web_platform/src/tuning/TuningApp.tsx +++ b/web_platform/src/tuning/TuningApp.tsx @@ -1,39 +1,14 @@ -import { useEffect, useMemo, useState, type ReactNode } from 'react'; -import { - Bot, - Check, - Download, - FlaskConical, - Pause, - Play, - RefreshCw, - Square, - Upload, - X, -} from 'lucide-react'; -import { Badge, Button, ProgressBar, Select } from '../components/ui'; +import { useEffect, useState, type ReactNode } from 'react'; +import { Bot, FlaskConical, Play, RefreshCw, Wifi } from 'lucide-react'; +import { useShallow } from 'zustand/react/shallow'; +import { Badge, Button, Select } from '../components/ui'; import { LocalTrainingClient } from '../training/LocalTrainingClient'; -import { - DEFAULT_TRAINING_ENDPOINT, - localStored, - rememberTrainingConnection, - sessionStored, - TRAINING_ENDPOINT_KEY, - TRAINING_TOKEN_KEY, - TUNING_SESSION_KEY, -} from '../training/storage'; -import type { - ObjectiveWeights, - ScalarSeries, - TuningCapability, - TuningMode, - TuningProposal, - TuningSession, - TuningTrial, -} from '../training/types'; -import { ScalarChart } from './ScalarChart'; +import { rememberTrainingConnection } from '../training/storage'; +import type { ObjectiveWeights, TuningCreateRequest, TuningMode } from '../training/types'; +import { OBJECTIVE_META, STATE_META } from './domain'; +import { TuningConsole } from './TuningConsole'; +import { useTuningStore } from './tuningStore'; -const ACTIVE = new Set(['queued', 'running', 'evaluating', 'awaiting_approval', 'paused']); const DEFAULT_OBJECTIVES: ObjectiveWeights = { velocity_tracking: 0.35, action_smoothness: 0.2, @@ -42,79 +17,29 @@ const DEFAULT_OBJECTIVES: ObjectiveWeights = { foot_slip: 0.1, energy: 0.05, }; -const OBJECTIVE_LABELS: Record = { - velocity_tracking: '速度跟踪', - action_smoothness: '动作平滑', - posture_stability: '姿态稳定', - fall_avoidance: '减少跌倒', - foot_slip: '足端滑移', - energy: '能耗', -}; - -function errorText(value: unknown): string { - return value instanceof Error ? value.message : String(value); -} -function stateLabel(value: string): string { - return ( - { - queued: '排队', - running: '训练中', - evaluating: '评估中', - awaiting_approval: '等待审批', - paused: '已暂停', - interrupted: '已中断', - succeeded: '已完成', - failed: '失败', - cancelled: '已取消', - completed: '完成', - }[value] ?? value - ); -} -function downloadFile(file: File): void { - const url = URL.createObjectURL(file); - const anchor = document.createElement('a'); - anchor.href = url; - anchor.download = file.name; - anchor.click(); - URL.revokeObjectURL(url); -} -function downloadJson(name: string, value: unknown): void { - downloadFile( - new File([JSON.stringify(value, null, 2) + '\n'], name, { type: 'application/json' }), - ); -} export function TuningApp() { - const [endpoint, setEndpoint] = useState(() => - localStored(TRAINING_ENDPOINT_KEY, DEFAULT_TRAINING_ENDPOINT), + const [endpoint, token, capability, connectionState, sessionId, error] = useTuningStore( + useShallow( + (state) => + [ + state.endpoint, + state.token, + state.capability, + state.connectionState, + state.sessionId, + state.error, + ] as const, + ), ); - const [token, setToken] = useState(() => sessionStored(TRAINING_TOKEN_KEY)); - const [capability, setCapability] = useState(); - const [sessions, setSessions] = useState([]); - const [session, setSession] = useState(); - const [selectedTrialId, setSelectedTrialId] = useState(); - const [series, setSeries] = useState([]); - const [busy, setBusy] = useState(false); - const [error, setError] = useState(); - const [mode, setMode] = useState('approval'); - const [runName, setRunName] = useState('go2-auto-tune'); - const [numEnvs, setNumEnvs] = useState(4096); - const [trialCount, setTrialCount] = useState(12); - const [gpuIds, setGpuIds] = useState('0'); - const [fallbackEnabled, setFallbackEnabled] = useState(false); - const [objectives, setObjectives] = useState(DEFAULT_OBJECTIVES); - const [smoothing, setSmoothing] = useState(0.3); - const [tagFilter, setTagFilter] = useState(''); - const [feedback, setFeedback] = useState(''); - const [patchText, setPatchText] = useState(''); + const [testingAgent, setTestingAgent] = useState(false); useEffect(() => { const receive = (event: MessageEvent) => { if (event.origin !== location.origin || typeof event.data !== 'object') return; const data = event.data as { type?: string; endpoint?: string; token?: string }; if (data.type === 'mujoco-tuning-credentials' && data.endpoint && data.token) { - setEndpoint(data.endpoint); - setToken(data.token); + useTuningStore.getState().setConnection(data.endpoint, data.token); rememberTrainingConnection(data.endpoint, data.token); } }; @@ -123,693 +48,293 @@ export function TuningApp() { return () => window.removeEventListener('message', receive); }, []); - const client = () => new LocalTrainingClient(endpoint, token); - const connect = async () => { - setBusy(true); - setError(undefined); + const testAgent = async () => { + setTestingAgent(true); + useTuningStore.getState().clearError(); try { - const api = client(); - const [nextCapability, nextSessions] = await Promise.all([ - api.tuningCapability(), - api.tuningSessions(), - ]); - setCapability(nextCapability); - setSessions(nextSessions); - rememberTrainingConnection(api.endpoint, api.token); - const remembered = localStored(TUNING_SESSION_KEY); - const target = nextSessions.find((item) => item.id === remembered) ?? nextSessions[0]; - if (target) { - const detail = await api.tuningSession(target.id); - setSession(detail); - setSelectedTrialId(detail.currentTrialId ?? detail.bestTrialId ?? detail.trials.at(-1)?.id); - } + await new LocalTrainingClient(endpoint, token).testTuningAgent(); } catch (value) { - setError(errorText(value)); + useTuningStore.setState({ error: value instanceof Error ? value.message : String(value) }); } finally { - setBusy(false); + setTestingAgent(false); } }; - const sessionId = session?.id; - const sessionState = session?.state; - useEffect(() => { - if (!sessionId || !sessionState || !ACTIVE.has(sessionState)) return; - const timer = window.setInterval(() => { - void new LocalTrainingClient(endpoint, token) - .tuningSession(sessionId) - .then((next) => { - setSession(next); - setSelectedTrialId((current) => current ?? next.currentTrialId ?? next.trials.at(-1)?.id); - }) - .catch((value: unknown) => setError(errorText(value))); - }, 2000); - return () => window.clearInterval(timer); - }, [endpoint, sessionId, sessionState, token]); - - useEffect(() => { - if (!sessionId || !selectedTrialId) return; - let disposed = false; - const refresh = () => - new LocalTrainingClient(endpoint, token) - .tuningMetrics(sessionId, selectedTrialId, [], 1200) - .then((value) => { - if (!disposed) setSeries(value.series); - }) - .catch((value: unknown) => { - if (!disposed) setError(errorText(value)); - }); - void refresh(); - const timer = window.setInterval(() => void refresh(), 3000); - return () => { - disposed = true; - window.clearInterval(timer); - }; - }, [endpoint, selectedTrialId, sessionId, token]); - - const start = async () => { - setBusy(true); - setError(undefined); - try { - const ids = gpuIds - .split(/[\s,]+/) - .filter(Boolean) - .map(Number); - if (!ids.length || ids.some((value) => !Number.isInteger(value) || value < 0)) - throw new Error('GPU 编号必须是非负整数'); - const next = await client().startTuning({ - taskId: 'Unitree-Go2-Flat', - mode, - runName, - numEnvs, - seed: 42, - gpuIds: ids, - trialCount, - initialIterations: 300, - middleIterations: 900, - finalIterations: 2000, - evalNumEnvs: 256, - evalSteps: 1000, - objectiveWeights: objectives, - fallbackEnabled, - }); - setSession(next); - setSelectedTrialId(next.currentTrialId ?? next.trials[0]?.id); - localStorage.setItem(TUNING_SESSION_KEY, next.id); - } catch (value) { - setError(errorText(value)); - } finally { - setBusy(false); - } - }; - - const runAction = async (action: 'pause' | 'resume' | 'cancel') => { - if (!session) return; - setBusy(true); - try { - setSession( - action === 'cancel' - ? await client().cancelTuning(session.id) - : await client().tuningAction(session.id, action), - ); - } catch (value) { - setError(errorText(value)); - } finally { - setBusy(false); - } - }; - - const pending = session?.proposals - .slice() - .reverse() - .find((item) => item.state === 'pending'); - const decide = async (proposal: TuningProposal, action: 'approve' | 'reject') => { - if (!session) return; - setBusy(true); - try { - const payload: { feedback?: string; patch?: TuningProposal['patch'] } = { feedback }; - if (action === 'approve') - payload.patch = JSON.parse( - patchText || JSON.stringify(proposal.patch), - ) as TuningProposal['patch']; - setSession(await client().decideProposal(session.id, proposal.id, action, payload)); - setFeedback(''); - setPatchText(''); - } catch (value) { - setError(errorText(value)); - } finally { - setBusy(false); - } - }; - - const completed = session?.trials.filter((trial) => trial.state === 'completed').length ?? 0; - const totalStages = session - ? session.config.trialCount + session.config.promote.slice(1).reduce((a, b) => a + b, 0) - : 1; - const selectedTrial = session?.trials.find((trial) => trial.id === selectedTrialId); - const filteredSeries = useMemo( - () => series.filter((item) => item.tag.toLowerCase().includes(tagFilter.toLowerCase())), - [series, tagFilter], - ); - return ( -
-
+
+

- Go2 奖励函数自调参 Agent + + + + Go2 奖励函数自调参 Agent

-

- DeepSeek 建议 · 固定协议评估 · TensorBoard Scalars +

+ MuJoCo WASM · Local PPO · ASHA 300→900→2000

{capability && ( - {capability.model} · {capability.configured ? '已配置' : '未配置'} + {capability.model} ·{' '} + {capability.configured ? 'Agent Online' : '未配置'} )}
-
- - setEndpoint(event.target.value)} - /> - - - setToken(event.target.value)} - /> - - -
- - {!session ? ( - { - const next = await client().tuningSession(id); - setSession(next); - setSelectedTrialId(next.currentTrialId ?? next.bestTrialId ?? next.trials.at(-1)?.id); - }} - /> - ) : ( -
- - -
-
- - setTagFilter(event.target.value)} - placeholder="Episode_Reward / Evaluation" - /> - - - setSmoothing(Number(event.target.value))} - /> - - - {selectedTrial - ? `Trial ${selectedTrial.number} / rung ${selectedTrial.rung}` - : '请选择 trial'} - -
- - {selectedTrial?.evaluation && } - -
- - -
+ + + + useTuningStore.getState().setConnection(endpoint, event.target.value) + } + /> + + + )} + + {sessionId ? : } + {error && ( -
useTuningStore.getState().clearError()} > {error} -
+ )} +
+ ); +} + +function NewSessionView() { + const [mode, setMode] = useState('approval'); + const [runName, setRunName] = useState('go2-auto-tune'); + const [numEnvs, setNumEnvs] = useState(4096); + const [trialCount, setTrialCount] = useState(12); + const [gpuIds, setGpuIds] = useState('0'); + const [fallbackEnabled, setFallbackEnabled] = useState(false); + const [objectives, setObjectives] = useState(DEFAULT_OBJECTIVES); + const [sessions, connectionState, capability] = useTuningStore( + useShallow((state) => [state.sessions, state.connectionState, state.capability] as const), + ); + const busy = connectionState === 'connecting'; + const objectiveTotal = Object.values(objectives).reduce((sum, value) => sum + value, 0); + + const start = async () => { + const ids = gpuIds + .split(/[\s,]+/) + .filter(Boolean) + .map(Number); + if (!ids.length || ids.some((value) => !Number.isInteger(value) || value < 0)) { + useTuningStore.setState({ error: 'GPU 编号必须是非负整数' }); + return; + } + const request: TuningCreateRequest = { + taskId: 'Unitree-Go2-Flat', + mode, + runName, + numEnvs, + seed: 42, + gpuIds: ids, + trialCount, + initialIterations: 300, + middleIterations: 900, + finalIterations: 2000, + evalNumEnvs: 256, + evalSteps: 1000, + objectiveWeights: objectives, + fallbackEnabled, + }; + await useTuningStore.getState().startSession(request); + }; + + return ( +
+
+
+
+
+

新建 Unitree-Go2-Flat 调参 Session

+

+ 白名单 Reward Patch · 3-seed 固定协议评估 · 非破坏性 Preset +

+
+ RTX 5080 Profile +
+
+ + setRunName(event.target.value)} + /> + + + + + + + + setGpuIds(event.target.value)} + /> + + +
+

+ 调参次数包含基线;连续 4 个建议无显著提升时提前停止。默认逐轮审批不会跳过基线。 +

+ +
+

六维目标权重

+ + Σ {(objectiveTotal * 100).toFixed(0)}% + +
+
+ {OBJECTIVE_META.map(({ key, label }) => ( + + ))} +
+ +
+ +
+
+

历史 Sessions

+ {sessions.length} +
+
+ {sessions.map((session) => ( + + ))} + {!sessions.length && ( +
+ 连接训练服务后显示历史会话 +
+ )} +
+
+
); } -function NewSessionForm(props: { - mode: TuningMode; - setMode(value: TuningMode): void; - runName: string; - setRunName(value: string): void; - numEnvs: number; - setNumEnvs(value: number): void; - trialCount: number; - setTrialCount(value: number): void; - gpuIds: string; - setGpuIds(value: string): void; - fallback: boolean; - setFallback(value: boolean): void; - objectives: ObjectiveWeights; - setObjectives(value: ObjectiveWeights): void; - start(): Promise; - busy: boolean; - sessions: TuningSession[]; - open(id: string): Promise; -}) { - return ( -
-
-

新建 Unitree-Go2-Flat 调参 Session

-
- - props.setRunName(e.target.value)} - /> - - - - - - - - props.setGpuIds(e.target.value)} - /> - - -
-

目标权重

-
- {(Object.keys(props.objectives) as (keyof ObjectiveWeights)[]).map((key) => ( - - ))} -
-

a + b, 0) - 1) < 1e-6 ? 'text-success' : 'text-danger'}`} - > - 总和:{Math.round(Object.values(props.objectives).reduce((a, b) => a + b, 0) * 100)}% -

- -
-
-

历史 Sessions

-
- {props.sessions.map((item) => ( - - ))} -
-
-
- ); -} - -function EvaluationCard({ trial }: { trial: TuningTrial }) { - return ( -
-

固定协议评估

-
- {Object.entries(trial.evaluation?.metrics ?? {}).map(([name, value]) => ( -
-

- {name} -

-

{value.toFixed(5)}

-
- ))} -
-
- ); -} -function Leaderboard({ trials, select }: { trials: TuningTrial[]; select(id: string): void }) { - const ranked = [...trials] - .filter((trial) => trial.score !== undefined && trial.score !== null) - .sort((a, b) => (b.score ?? -999) - (a.score ?? -999)); - return ( -
-

排行榜

-
- - - - - - - - - - - - {ranked.map((trial, index) => ( - select(trial.id)} - > - - - - - - - ))} - -
排名TrialRung分数安全门槛
{index + 1}T{trial.number}{trial.rung}{trial.score?.toFixed(5)}{trial.eligible ? '通过' : '未通过'}
-
-
- ); -} -function ApprovalCard(props: { - proposal: TuningProposal; - patchText: string; - setPatchText(value: string): void; - feedback: string; - setFeedback(value: string): void; - decide(proposal: TuningProposal, action: 'approve' | 'reject'): Promise; - busy: boolean; -}) { - return ( -
-
-

等待审批

- 置信度 {Math.round(props.proposal.confidence * 100)}% -
-

{props.proposal.rationale}

-