feat: release v0.4 with textured Microban visuals

This commit is contained in:
2026-10-29 15:55:17 +08:00
parent 09f7b02c3e
commit 57b3108ab3
37 changed files with 911 additions and 15 deletions
+2 -2
View File
@@ -20,12 +20,12 @@ The application opens directly into simulation, defaults to Microduck, and keeps
| Robot | Default scene / presentation | Controls |
|---|---|---|
| Microduck | Shared 4.5 × 4.5 m bounded arena, ball, original GLB/palette/comic effects | Existing walking, skating, skills and recovery |
| Microban | Robot-scaled 5.5 × 5.5 m arena, dark/orange grid and comic UI, MJCF robot visuals | Walking policy + independent head rotation |
| Microban | Robot-scaled 5.5 × 5.5 m arena, dark/orange grid and comic UI, textured per-body GLB visuals | Walking policy + independent head rotation |
| RP1 | Robot-scaled 22 × 22 m arena/comic UI, original 24-DOF MJCF visuals | Completed walking policy; trained 10 s dance segment (J) and full 4.36 s wave (H) |
Switch robots using **Robot** in the toolbar. Returning to Microduck restores the session's colour and feet / rollers choice, but initializes a fresh physical state. Failed loads keep the previous robot available and show an inline retry action.
Microduck policies / assets originate from [pollen-robotics/microduck](https://github.com/pollen-robotics/microduck) and [pollen-robotics/microduck_rl](https://github.com/pollen-robotics/microduck_rl). Microban assets originate from [Rhoban/microban](https://github.com/Rhoban/microban); exact source hashes, dirty-worktree provenance and license notices are included in its Package.
Microduck policies / assets originate from [pollen-robotics/microduck](https://github.com/pollen-robotics/microduck) and [pollen-robotics/microduck_rl](https://github.com/pollen-robotics/microduck_rl). Microban physics / policy assets originate from [Rhoban/microban](https://github.com/Rhoban/microban); its textured visual replacement comes from the user-supplied `microban2.zip`. Exact hashes, dirty-worktree provenance and license caveats are included in its [Package source notes](app/public/robots/microban/SOURCE.md).
## Keyboard and mouse
+212
View File
@@ -0,0 +1,212 @@
import { test, expect } from "@playwright/test";
async function boot(page) {
await page.goto("/");
await expect(
page.getByRole("button", { name: "Pause", exact: true }),
).toBeEnabled();
await page.evaluate(() => window.__gameApi.pause());
}
test("20 textured GLBs replace only Microban visuals, align to original geometry and follow all bodies", async ({
page,
}, info) => {
const errors = [];
page.on("pageerror", (error) => errors.push(error.message));
await boot(page);
expect(
await page.evaluate(() => window.__gameApi.switchRobot("microban")),
).toBe(true);
const inspect = () =>
page.evaluate(() => {
const r = window.rl,
layer = r.scene.getObjectByName("body-visuals"),
rig = layer.parent;
rig.updateMatrixWorld(true);
let maxFrameError = 0,
maxBoundsError = 0,
maps = 0,
colorMaps = 0,
meshes = 0,
hidden = 0;
const bounds = (objects, frame) => {
const lo = [Infinity, Infinity, Infinity],
hi = [-Infinity, -Infinity, -Infinity];
const inverse = frame.matrixWorld.clone().invert(),
vector = frame.position.clone();
for (const mesh of objects) {
const transform = inverse.clone().multiply(mesh.matrixWorld),
attr = mesh.geometry.attributes.position;
for (let i = 0; i < attr.count; i++) {
vector.fromBufferAttribute(attr, i).applyMatrix4(transform);
const v = vector.toArray();
for (let axis = 0; axis < 3; axis++) {
lo[axis] = Math.min(lo[axis], v[axis]);
hi[axis] = Math.max(hi[axis], v[axis]);
}
}
}
return [...lo, ...hi];
};
for (const frame of layer.children) {
const body = frame.name.slice("body-visual:".length),
id = r.mujoco.mj_name2id(
r.model,
r.mujoco.mjtObj.mjOBJ_BODY.value,
body,
);
const p = Array.from(r.data.xpos.slice(id * 3, id * 3 + 3));
const q = Array.from(r.data.xquat.slice(id * 4, id * 4 + 4));
maxFrameError = Math.max(
maxFrameError,
...p.map((v, i) => Math.abs(v - frame.position.toArray()[i])),
...[q[1], q[2], q[3], q[0]].map((v, i) =>
Math.abs(v - frame.quaternion.toArray()[i]),
),
);
const originals = rig.children.filter(
(mesh) =>
mesh.userData.geomId !== undefined &&
r.model.geom_bodyid[mesh.userData.geomId] === id,
);
const replacements = [];
frame.traverse((object) => {
if (!object.isMesh) return;
meshes++;
replacements.push(object);
const materials = [object.material].flat();
maps += materials.filter(
(material) => material.normalMap?.image?.width > 0,
).length;
colorMaps += materials.filter(
(material) => material.map?.image?.width > 0,
).length;
});
hidden += originals.filter((mesh) => !mesh.visible).length;
const oldBounds = bounds(originals, frame),
newBounds = bounds(replacements, frame);
maxBoundsError = Math.max(
maxBoundsError,
...oldBounds.map((v, i) => Math.abs(v - newBounds[i])),
);
}
return {
count: layer.children.length,
maps,
colorMaps,
meshes,
hidden,
maxFrameError,
maxBoundsError,
obs: r.buildObs().length,
};
});
const initial = await inspect();
console.log("Microban visual alignment", initial);
expect(initial.count).toBe(20);
expect(initial.maps).toBeGreaterThanOrEqual(20);
expect(initial.colorMaps).toBeGreaterThanOrEqual(1);
expect(initial.hidden).toBeGreaterThan(20);
expect(initial.maxFrameError).toBeLessThan(1e-12);
expect(initial.maxBoundsError).toBeLessThan(0.0031);
expect(initial.obs).toBe(63);
// Change root yaw and every hinge, including head: no bind-pose-only rig.
await page.evaluate(() => {
const r = window.rl;
r.data.qpos[3] = Math.cos(0.4);
r.data.qpos[6] = Math.sin(0.4);
for (let i = 7; i < r.model.nq; i++) r.data.qpos[i] += 0.12;
r.mujoco.mj_forward(r.model, r.data);
});
await page.waitForTimeout(150);
const articulated = await inspect();
expect(articulated.maxFrameError).toBeLessThan(1e-12);
expect(articulated.maxBoundsError).toBeLessThan(0.0031);
await page.evaluate(() => window.rl.resetSim());
await page.waitForTimeout(150);
// Front 3/4 view for a useful visual acceptance artifact.
await page.evaluate(() => {
window.rl.camera.position.set(0.58, 0.38, 0.58);
window.rl.controls.target.set(0, 0.17, 0);
window.rl.controls.update();
});
await page.waitForTimeout(200);
const screenshot = info.outputPath("textured-microban.png");
await page.screenshot({ path: screenshot });
await info.attach("textured-microban.png", {
path: screenshot,
contentType: "image/png",
});
expect(errors).toEqual([]);
});
test("failed visual candidate rolls back without hiding the old robot; retry loads textures", async ({
page,
}) => {
await boot(page);
const before = await page.evaluate(
() => window.__gameApi.diagnostics().sceneChildren,
);
let fail = true;
await page.route(
"**/robots/microban/visuals/LINK10__visual_0__LINK10.glb",
(route) =>
fail
? route.fulfill({ status: 404, body: "missing visual" })
: route.continue(),
);
expect(
await page.evaluate(() => window.__gameApi.switchRobot("microban")),
).toBe(false);
expect(await page.evaluate(() => window.__store.getState().robotId)).toBe(
"microduck-legs",
);
expect(
await page.evaluate(() => window.__gameApi.diagnostics().sceneChildren),
).toBe(before);
expect(
await page.evaluate(
() => window.rl.scene.getObjectByName("body-visuals") === undefined,
),
).toBe(true);
fail = false;
expect(
await page.evaluate(() => window.__gameApi.switchRobot("microban")),
).toBe(true);
expect(
await page.evaluate(
() => window.rl.scene.getObjectByName("body-visuals").children.length,
),
).toBe(20);
});
test("visual replacement leaves deterministic physics and ONNX actions unchanged", async ({
page,
}) => {
await boot(page);
let textured = false;
await page.route("**/robots/microban/robot.json", async (route) => {
const response = await route.fetch(),
config = await response.json();
if (!textured) delete config.render.body_visuals;
await route.fulfill({ json: config });
});
async function run() {
expect(
await page.evaluate(() => window.__gameApi.switchRobot("microban")),
).toBe(true);
return page.evaluate(async () => {
const r = window.rl;
await r.resetSim();
await r.step(80);
return {
qpos: Array.from(r.data.qpos),
qvel: Array.from(r.data.qvel),
action: Array.from(r.lastAction),
};
});
}
const white = await run();
expect(
await page.evaluate(() => window.__gameApi.switchRobot("microduck-legs")),
).toBe(true);
textured = true;
const visual = await run();
expect(visual).toEqual(white);
});
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "microduck-sandbox",
"version": "0.3.0",
"version": "0.4.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "microduck-sandbox",
"version": "0.3.0",
"version": "0.4.0",
"dependencies": {
"@emotion/react": "^11.14.0",
"@emotion/styled": "^11.14.1",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "microduck-sandbox",
"private": true,
"version": "0.3.0",
"version": "0.4.0",
"type": "module",
"scripts": {
"dev": "vite",
+8
View File
@@ -11,4 +11,12 @@ The precise origin remote, HEAD, dirty-worktree flag, per-file SHA-256 and behav
Browser adaptations (not edits to the copied XML): 0.005 s physics timestep; separate flat ground; explicit 18-joint policy order excluding head; body-frame gravity and IMU-site gyroscope following source observation semantics; limited joint-PD torque control instead of BAM. Default PD kp=0.3 Nm/rad, kd=0.02 Nm·s/rad, torque limit=0.65 Nm are experimentally selected simplified parameters, not metadata gains and not a claim of hardware equivalence.
## Textured visual replacement
`visuals/*.glb` are the 20 files copied byte-for-byte from the user-supplied `microban2.zip`, replacing the earlier `microban.zip` visual export, not from the upstream checkout above. They retain their embedded PBR materials and 46 textures (45 normal maps and one screen base-colour image, all 1024 × 1024). The archive contains no license notice; **confirm public redistribution rights with the asset owner** rather than assuming the upstream model license covers these replacements. Archive and file SHA-256 values are recorded separately in `visual-source-manifest.json`.
Reimport with `python3 app/scripts/import-microban-visuals.py /path/to/microban2.zip`. This does not change `model.xml`, the original STL files, policy, original source manifest or the reviewed `render.body_visuals` mapping in `robot.json`.
The GLBs have different local frames from the original MJCF. `robot.json` records fixed, visual-only rigid transforms (metres and xyzw quaternions), checked against the original per-body geometry; body-local bounding-box endpoint differences are at most 3.01 mm, allowing the replacement's added details. Mapping: BASE → trunk; LINK1 → head; LINK2–4 → right arm; LINK5–7 → left arm; LINK8–13 → left leg; LINK14–19 → right leg. Vertex positions and node transforms in all 20 GLBs are unchanged from the previous visual export, so the existing body registration is retained. Runtime follows the existing MuJoCo body poses and hides only the replaced visual geoms; collisions, inertias, joints, policy inputs and actuation are untouched. This is geometric registration, not a claim that the archive supplied authoritative joint-frame metadata.
No Placo, real-robot control, source Python code, CAD project, BAM battery model or communication delays are bundled into the browser runtime.
+113
View File
@@ -315,6 +315,119 @@
},
"render": {
"source": "mjcf_visual",
"body_visuals": [
{
"body": "trunk",
"path": "visuals/BASE__visual_0__BASE.glb",
"quaternion": [0.7071067811865476, 0, 0, 0.7071067811865476]
},
{
"body": "head",
"path": "visuals/LINK1__visual_0__LINK1.glb",
"quaternion": [0.5, -0.5, -0.5, 0.5]
},
{
"body": "shoulder",
"path": "visuals/LINK2__visual_0__LINK2.glb"
},
{
"body": "humerus__configuration_right",
"path": "visuals/LINK3__visual_0__LINK3.glb",
"position": [0, 0, -0.02755],
"quaternion": [0, -0.7071067811865476, 0, 0.7071067811865476]
},
{
"body": "radius",
"path": "visuals/LINK4__visual_0__LINK4.glb",
"position": [-0.00529885, -0.00021476, 0.00145]
},
{
"body": "shoulder_2",
"path": "visuals/LINK5__visual_0__LINK5.glb"
},
{
"body": "humerus__configuration_left",
"path": "visuals/LINK6__visual_0__LINK6.glb",
"position": [-0.00529885, -0.00021476, -0.02755],
"quaternion": [0.7071067811865476, 0, -0.7071067811865476, 0]
},
{
"body": "radius_2",
"path": "visuals/LINK7__visual_0__LINK7.glb",
"position": [-0.00529885, 0.00021476, -0.00145],
"quaternion": [0, 0, 1, 0]
},
{
"body": "hip_2",
"path": "visuals/LINK8__visual_0__LINK8.glb",
"quaternion": [0, 0.7071067811865476, 0.7071067811865476, 0]
},
{
"body": "hip_block__configuration_left",
"path": "visuals/LINK9__visual_0__LINK9.glb",
"position": [0, 0, 0.00027893],
"quaternion": [0.7071067811865476, 0, 0.7071067811865476, 0]
},
{
"body": "femur__configuration_left",
"path": "visuals/LINK10__visual_0__LINK10.glb",
"position": [0, 0, -0.00145],
"quaternion": [1, 0, 0, 0]
},
{
"body": "tibia__configuration_left",
"path": "visuals/LINK11__visual_0__LINK11.glb",
"position": [0, 0, -0.00145],
"quaternion": [1, 0, 0, 0]
},
{
"body": "ankle_block__configuration_right",
"path": "visuals/LINK12__visual_0__LINK12.glb",
"position": [0, 0, -0.00145],
"quaternion": [0, 1, 0, 0]
},
{
"body": "foot_2",
"path": "visuals/LINK13__visual_0__LINK13.glb",
"position": [0, 0, 0.0008],
"quaternion": [0, -0.7071067811865476, 0, 0.7071067811865476]
},
{
"body": "hip",
"path": "visuals/LINK14__visual_0__LINK14.glb",
"quaternion": [0.7071067811865476, 0, 0, 0.7071067811865476]
},
{
"body": "hip_block__configuration_right",
"path": "visuals/LINK15__visual_0__LINK15.glb",
"position": [0, 0, 0.0008],
"quaternion": [0.7071067811865476, 0, 0.7071067811865476, 0]
},
{
"body": "femur__configuration_right",
"path": "visuals/LINK16__visual_0__LINK16.glb",
"position": [0, 0, -0.02755],
"quaternion": [0, 1, 0, 0]
},
{
"body": "tibia__configuration_right",
"path": "visuals/LINK17__visual_0__LINK17.glb",
"position": [0, 0, -0.02745],
"quaternion": [0, 1, 0, 0]
},
{
"body": "ankle_block__configuration_left",
"path": "visuals/LINK18__visual_0__LINK18.glb",
"position": [0, 0, -0.02755],
"quaternion": [1, 0, 0, 0]
},
{
"body": "foot",
"path": "visuals/LINK19__visual_0__LINK19.glb",
"position": [0, 0, 0.0008],
"quaternion": [0, -0.7071067811865476, 0, 0.7071067811865476]
}
],
"groups": [
2
],
@@ -0,0 +1,87 @@
{
"source": "User-provided microban2.zip (textured visual replacement)",
"archive_sha256": "dfb20a34103afecdc7996cc5ac9f6a61705560717f9c6aac420f3d6307d3e6d4",
"license": "Not supplied in the archive; confirm redistribution rights with the asset owner.",
"files": [
{
"file": "visuals/BASE__visual_0__BASE.glb",
"sha256": "32cc5d7df7ab62779cb61ae777dafe485fc357489998020fba86e2bb94d968a6"
},
{
"file": "visuals/LINK1__visual_0__LINK1.glb",
"sha256": "33f7776ad30bc04fb0a93a1a80641636d5cca382c12093bf97df9d9cd1416553"
},
{
"file": "visuals/LINK2__visual_0__LINK2.glb",
"sha256": "ffbdd241b96d8e9d371bbddb57908912e95edb4c7ce88e9e20a79c663f80058f"
},
{
"file": "visuals/LINK3__visual_0__LINK3.glb",
"sha256": "01718a43c6b8936543c6c2e47b00150907c7ac7c892c90f9d612a21d6fc566b9"
},
{
"file": "visuals/LINK4__visual_0__LINK4.glb",
"sha256": "7dad340b731e9f5ed8fe37bc77dd2b7b5ff1ac540cc501dad35b487438ab4e0d"
},
{
"file": "visuals/LINK5__visual_0__LINK5.glb",
"sha256": "aa54506ab2503ac34cb4d8240772798d353f72a04b81224543b9347df38a68ba"
},
{
"file": "visuals/LINK6__visual_0__LINK6.glb",
"sha256": "2bed471a7c3e51107d5307130c342696b3e986384584c4891b4b7a161380aeb4"
},
{
"file": "visuals/LINK7__visual_0__LINK7.glb",
"sha256": "909cc20e48dad6dd8478c8dd19499ffc49c8427305ead62e1b267ae8a3939d9e"
},
{
"file": "visuals/LINK8__visual_0__LINK8.glb",
"sha256": "d2a43a6606906fd8bd8a982208ea2121245969bded44a3f81ee074e1968c5abf"
},
{
"file": "visuals/LINK9__visual_0__LINK9.glb",
"sha256": "33f34ba76488c0dfe99bcf80bb48ff3ba0c0240e506a79203d380e20aa17f952"
},
{
"file": "visuals/LINK10__visual_0__LINK10.glb",
"sha256": "4bf30a00a35ec588a74476cefc6cff79b14f16ebf8136a55ab8392c8b2b5a6be"
},
{
"file": "visuals/LINK11__visual_0__LINK11.glb",
"sha256": "04e8b462a92a6689074211bc1243a28bd031148c6f390cf00d9af977e33d4616"
},
{
"file": "visuals/LINK12__visual_0__LINK12.glb",
"sha256": "e1549141dbd462ef4aa31be2f5bb8ae2f31a0d0b49da3de41911ef482c008cd7"
},
{
"file": "visuals/LINK13__visual_0__LINK13.glb",
"sha256": "ded693e8bc53cdda7ac36b5f4d49c56391eaa283f6549872e2eeaa33d0b43afd"
},
{
"file": "visuals/LINK14__visual_0__LINK14.glb",
"sha256": "03f70f51f86a4eb67b069629ed89576cae71910f903d34638dc8fd5fc8b9bb4d"
},
{
"file": "visuals/LINK15__visual_0__LINK15.glb",
"sha256": "87202389c4bc6dade765d805f399f8577add3c2b3a10a317264dbe4e4e99f54d"
},
{
"file": "visuals/LINK16__visual_0__LINK16.glb",
"sha256": "eb8a287ca42899b728973690352974a9351fcbc7a7f9e7e632376f4aebdad3d0"
},
{
"file": "visuals/LINK17__visual_0__LINK17.glb",
"sha256": "1f942e545727ce342c97827f06d33d6ab8186120ea6dbcf36369eb2b4e3e54e5"
},
{
"file": "visuals/LINK18__visual_0__LINK18.glb",
"sha256": "808675acb0fddbbacdbed43ad71de1df7ffd0b898a8bf7814098cbe7d1d1ad29"
},
{
"file": "visuals/LINK19__visual_0__LINK19.glb",
"sha256": "b00cec2f4a0924efd7805f405ab0bb103e1ecdf1d56310caa3c275c7ef0a7c1b"
}
]
}
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Import the user-supplied, self-contained Microban visual GLBs only.
Usage: python3 app/scripts/import-microban-visuals.py /path/to/microban2.zip
Does not modify MJCF, policies or the reviewed body transforms in robot.json.
"""
import hashlib
import json
from pathlib import Path
import struct
import sys
import zipfile
def import_visuals(archive):
target = Path(__file__).resolve().parents[1] / "public/robots/microban"
links = ["BASE", *(f"LINK{i}" for i in range(1, 20))]
names = [f"{link}__visual_0__{link}.glb" for link in links]
with zipfile.ZipFile(archive) as source:
if sorted(source.namelist()) != sorted(names):
raise ValueError("Expected exactly BASE and LINK1..LINK19 GLBs")
files = {name: source.read(name) for name in names}
# Validate everything before copying any file. Never extract ZIP paths.
for name, data in files.items():
magic, version, length, json_length, chunk_type = struct.unpack_from(
"<5I", data
)
if (magic, version, length, chunk_type) != (
0x46546C67, 2, len(data), 0x4E4F534A
):
raise ValueError(f"{name}: invalid GLB")
gltf = json.loads(data[20:20 + json_length])
for asset in [*gltf.get("buffers", []), *gltf.get("images", [])]:
if "uri" in asset and not asset["uri"].startswith("data:"):
raise ValueError(f"{name}: external buffer/image is unsupported")
(target / "visuals").mkdir(exist_ok=True)
records = []
for name, data in files.items():
(target / "visuals" / name).write_bytes(data)
records.append({
"file": f"visuals/{name}",
"sha256": hashlib.sha256(data).hexdigest(),
})
manifest = {
"source": f"User-provided {archive.name} (textured visual replacement)",
"archive_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(),
"license": "Not supplied in the archive; confirm redistribution rights with the asset owner.",
"files": records,
}
(target / "visual-source-manifest.json").write_text(
json.dumps(manifest, indent=2) + "\n"
)
print(f"Imported {len(records)} visual GLBs; physics/policy files unchanged")
if __name__ == "__main__":
import_visuals(Path(sys.argv[1]))
+22 -10
View File
@@ -8,6 +8,7 @@ import {
} from "../src/platform/loader/SchemaValidator.js";
import { resolveWorldId } from "../src/platform/worlds/arena.js";
import { JointMotionReference } from "../src/platform/policy/JointMotionReference.js";
import { validateBodyGlb } from "../src/platform/render/BodyVisuals.js";
const root = fileURLToPath(new URL("../public/robots/", import.meta.url));
const entries = JSON.parse(await readFile(resolve(root, "index.json"), "utf8"));
for (const entry of entries) {
@@ -18,6 +19,7 @@ for (const entry of entries) {
const paths = [
config.model,
...config.assets.map((a) => a.path),
...(config.render.body_visuals ?? []).map((v) => v.path),
...config.skills
.filter((s) => s.controller.type === "onnx" && s.available !== false)
.map((s) => s.controller.policy),
@@ -39,6 +41,12 @@ for (const entry of entries) {
paths.push(skill.reference, spec.data);
}
for (const path of paths) await access(resolve(dirname(file), path));
for (const visual of config.render.body_visuals ?? []) {
const bytes = await readFile(resolve(dirname(file), visual.path));
validateBodyGlb(
bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength),
);
}
validateWorld(
JSON.parse(
await readFile(
@@ -47,15 +55,18 @@ for (const entry of entries) {
),
),
);
let provenance;
try {
provenance = JSON.parse(
await readFile(resolve(dirname(file), "source-manifest.json"), "utf8"),
);
} catch (error) {
if (error.code !== "ENOENT") throw error;
}
if (provenance)
let hasProvenance = false;
for (const name of ["source-manifest.json", "visual-source-manifest.json"]) {
let provenance;
try {
provenance = JSON.parse(
await readFile(resolve(dirname(file), name), "utf8"),
);
} catch (error) {
if (error.code !== "ENOENT") throw error;
continue;
}
hasProvenance = true;
for (const record of [
...provenance.files,
...(provenance.additionalNotices ?? []),
@@ -68,7 +79,8 @@ for (const entry of entries) {
`${entry.id}/${record.file}: source SHA-256 mismatch (check git lfs pull or document asset changes)`,
);
}
}
console.log(
`${config.id}: schema, World, ${paths.length} resource paths${provenance ? " and source hashes" : ""} OK`,
`${config.id}: schema, World, ${paths.length} resource paths${hasProvenance ? " and source hashes" : ""} OK`,
);
}
@@ -218,6 +218,21 @@ export function validatePackage(config) {
}
if (config.adapter === "generic" && config.render.source !== "mjcf_visual")
throw new Error("render.source: generic runtime requires mjcf_visual");
const visualBodies = new Set();
for (const visual of config.render.body_visuals ?? []) {
if (config.adapter !== "generic")
throw new Error("render.body_visuals: requires the generic runtime");
if (visualBodies.has(visual.body))
throw new Error(`render.body_visuals: duplicate body ${visual.body}`);
visualBodies.add(visual.body);
if (
visual.quaternion &&
Math.abs(Math.hypot(...visual.quaternion) - 1) > 1e-6
)
throw new Error(
"render.body_visuals: quaternion must have unit length (xyzw)",
);
}
return config;
}
export { schema as robotSchema };
+28
View File
@@ -379,6 +379,34 @@
"source": {
"enum": ["glb", "mjcf_visual"]
},
"body_visuals": {
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"required": ["body", "path"],
"additionalProperties": false,
"properties": {
"body": { "type": "string", "minLength": 1 },
"path": {
"type": "string",
"pattern": "^(?!/)(?!.*\\.\\.)[^\\\\:?#]+\\.glb$"
},
"position": {
"type": "array",
"minItems": 3,
"maxItems": 3,
"items": { "type": "number" }
},
"quaternion": {
"type": "array",
"minItems": 4,
"maxItems": 4,
"items": { "type": "number" }
}
}
}
},
"groups": {
"type": "array",
"items": {
+111
View File
@@ -0,0 +1,111 @@
import * as THREE from "three";
import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
import { fetchChecked } from "../loader/PackageLoader.js";
// Body GLBs must be self-contained: external image/buffer URIs would bypass
// PackageLoader's same-origin, hosting-prefix and signed-URL handling.
export function validateBodyGlb(buffer) {
const view = new DataView(buffer);
if (
buffer.byteLength < 20 ||
view.getUint32(0, true) !== 0x46546c67 ||
view.getUint32(4, true) !== 2 ||
view.getUint32(8, true) !== buffer.byteLength ||
view.getUint32(16, true) !== 0x4e4f534a ||
view.getUint32(12, true) > buffer.byteLength - 20
)
throw new Error("Body visual: invalid GLB (check git lfs pull)");
const json = JSON.parse(
new TextDecoder().decode(
new Uint8Array(buffer, 20, view.getUint32(12, true)),
),
);
for (const asset of [...(json.buffers ?? []), ...(json.images ?? [])])
if (asset.uri && !asset.uri.startsWith("data:"))
throw new Error("Body visual: GLB must embed all buffers and images");
return json;
}
// Visual-only layer. Nothing here changes the compiled model, qpos, collision
// geoms or policy inputs. Each attachment follows a MuJoCo body, not an
// independently animated skeleton or the compiler-recentered STL geom frame.
export class BodyVisuals {
static async load(mujoco, model, data, pkg) {
const specs = pkg.config.render.body_visuals ?? [];
const ids = specs.map(({ body }) => {
const id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_BODY.value, body);
if (id <= 0) throw new Error(`Body visual: missing robot body ${body}`);
return id;
});
const visuals = new BodyVisuals(data);
const loader = new GLTFLoader();
try {
// Sequential parsing keeps ownership unambiguous on a failed candidate:
// no outstanding parse may allocate resources after dispose() returns.
for (let i = 0; i < specs.length; i++) {
const spec = specs[i];
const bytes = await fetchChecked(pkg.asset(spec.path), "bytes");
validateBodyGlb(bytes);
const gltf = await loader.parseAsync(bytes, "");
// Retain every scene for cleanup, including non-default glTF scenes.
visuals.scenes.push(...gltf.scenes);
const frame = new THREE.Group();
frame.name = `body-visual:${spec.body}`;
gltf.scene.position.fromArray(spec.position ?? [0, 0, 0]);
gltf.scene.quaternion.fromArray(spec.quaternion ?? [0, 0, 0, 1]);
frame.add(gltf.scene);
visuals.root.add(frame);
visuals.items.push({ id: ids[i], frame });
}
visuals.sync();
return visuals;
} catch (error) {
visuals.dispose();
throw error;
}
}
constructor(data) {
this.data = data;
this.root = new THREE.Group();
this.root.name = "body-visuals";
this.items = [];
this.scenes = [];
}
sync() {
for (const { id, frame } of this.items) {
frame.position.fromArray(this.data.xpos, id * 3);
const a = id * 4,
q = this.data.xquat;
frame.quaternion.set(q[a + 1], q[a + 2], q[a + 3], q[a]);
}
}
dispose() {
if (this.disposed) return;
this.disposed = true;
const geometries = new Set(),
materials = new Set(),
textures = new Set(),
images = new Set();
for (const scene of this.scenes)
scene.traverse((object) => {
if (object.geometry) geometries.add(object.geometry);
for (const material of [object.material].flat().filter(Boolean)) {
materials.add(material);
for (const value of Object.values(material))
if (value?.isTexture) textures.add(value);
}
});
for (const geometry of geometries) geometry.dispose();
for (const material of materials) material.dispose();
for (const texture of textures) {
for (const image of [texture.source?.data].flat().filter(Boolean))
images.add(image);
texture.dispose();
}
for (const image of images) image.close?.();
this.root.removeFromParent();
this.root.clear();
this.scenes = [];
this.items = [];
}
}
+16
View File
@@ -1,5 +1,6 @@
import * as THREE from "three";
import { applyWorldGrid } from "./WorldRenderer.js";
import { BodyVisuals } from "./BodyVisuals.js";
// Geometry is compiled MuJoCo geometry, not raw STL: compiler-applied mesh
// scale/centering is already represented by vertices + geom transforms.
export class RigRenderer {
@@ -91,7 +92,21 @@ export class RigRenderer {
throw error;
}
}
async loadBodyVisuals(mujoco, pkg) {
if (!pkg.config.render.body_visuals?.length) return;
this.bodyVisuals = await BodyVisuals.load(
mujoco,
this.model,
this.data,
pkg,
);
const replaced = new Set(this.bodyVisuals.items.map(({ id }) => id));
for (const { id, mesh } of this.items)
if (replaced.has(this.model.geom_bodyid[id])) mesh.visible = false;
this.root.add(this.bodyVisuals.root);
}
sync() {
this.bodyVisuals?.sync();
const p = this.data.geom_xpos,
r = this.data.geom_xmat,
m = this.matrix;
@@ -120,6 +135,7 @@ export class RigRenderer {
}
}
dispose() {
this.bodyVisuals?.dispose();
for (const g of this.geometries.values()) g.dispose();
for (const m of this.materials) m.dispose();
this.geometries.clear();
+1
View File
@@ -89,6 +89,7 @@ export class RobotRuntime {
}
const { ctx, obs, actions, controller } = policies.get(primary.id);
rig = new RigRenderer(sim.model, sim.data, { ...config.render, world });
await rig.loadBodyVisuals(mujoco, pkg);
worldPresentation = await options.createWorldPresentation?.({
sim,
rig,
+139
View File
@@ -0,0 +1,139 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs/promises";
import * as THREE from "three";
import {
BodyVisuals,
validateBodyGlb,
} from "../src/platform/render/BodyVisuals.js";
import { validatePackage } from "../src/platform/loader/SchemaValidator.js";
const config = JSON.parse(
await fs.readFile(
new URL("../public/robots/microban/robot.json", import.meta.url),
),
);
function glb(json) {
const text = new TextEncoder().encode(JSON.stringify(json));
const buffer = new ArrayBuffer(20 + text.length);
const view = new DataView(buffer);
[0x46546c67, 2, buffer.byteLength, text.length, 0x4e4f534a].forEach(
(value, i) => view.setUint32(i * 4, value, true),
);
new Uint8Array(buffer, 20).set(text);
return buffer;
}
test("all 20 Microban body GLBs embed their materials/textures and stay visual-only", async () => {
validatePackage(config);
assert.equal(config.render.body_visuals.length, 20);
let textures = 0;
for (const { body, path } of config.render.body_visuals) {
assert.ok(!config.assets.some((asset) => asset.path === path));
const bytes = await fs.readFile(
new URL(`../public/robots/microban/${path}`, import.meta.url),
);
const json = validateBodyGlb(
bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength),
);
assert.ok(json.materials.length > 0);
if (body === "head")
assert.ok(
json.materials.some(
(material) => material.pbrMetallicRoughness?.baseColorTexture,
),
"microban2 screen colour texture must be preserved",
);
textures += json.images?.length ?? 0;
}
assert.ok(textures >= 20);
});
test("visual schema rejects duplicate bodies, non-unit quaternions and external/escaping paths", () => {
const duplicate = structuredClone(config);
duplicate.render.body_visuals.push(duplicate.render.body_visuals[0]);
assert.throws(() => validatePackage(duplicate), /duplicate body/);
const zero = structuredClone(config);
zero.render.body_visuals[0].quaternion = [0, 0, 0, 0];
assert.throws(() => validatePackage(zero), /unit length/);
for (const path of [
"https://host/a.glb",
"../a.glb",
"/a.glb",
"a.glb?x=1",
]) {
const invalid = structuredClone(config);
invalid.render.body_visuals[0].path = path;
assert.throws(() => validatePackage(invalid), /body_visuals/);
}
});
test("GLB validation fails closed on LFS pointers, truncation and external image/buffer requests", () => {
for (const bytes of [
new ArrayBuffer(0),
new TextEncoder().encode("version https://git-lfs.github.com/spec/v1")
.buffer,
glb({}).slice(0, -1),
])
assert.throws(() => validateBodyGlb(bytes), /invalid GLB/);
for (const json of [
{ images: [{ uri: "https://host/normal.png" }] },
{ buffers: [{ uri: "../buffer.bin" }] },
])
assert.throws(() => validateBodyGlb(glb(json)), /embed all/);
});
test("body attachments track MuJoCo body transforms without touching simulation arrays", () => {
const data = {
xpos: [0, 0, 0, 1, 2, 3],
xquat: [1, 0, 0, 0, 0.5, 0.5, 0.5, 0.5],
};
const before = structuredClone(data);
const visuals = new BodyVisuals(data),
frame = new THREE.Group();
visuals.items.push({ id: 1, frame });
visuals.sync();
assert.deepEqual(frame.position.toArray(), [1, 2, 3]);
assert.deepEqual(frame.quaternion.toArray(), [0.5, 0.5, 0.5, 0.5]);
assert.deepEqual(data, before);
});
test("shared GLB geometries, materials, textures and ImageBitmaps are released exactly once", () => {
const visuals = new BodyVisuals({}),
scene = new THREE.Group();
const geometry = new THREE.BoxGeometry(),
texture = new THREE.Texture();
let closed = 0;
texture.source.data = { close: () => closed++ };
const material = new THREE.MeshStandardMaterial({
map: texture,
normalMap: texture,
});
const counts = { geometry: 0, material: 0, texture: 0 };
geometry.addEventListener("dispose", () => counts.geometry++);
material.addEventListener("dispose", () => counts.material++);
texture.addEventListener("dispose", () => counts.texture++);
scene.add(
new THREE.Mesh(geometry, material),
new THREE.Mesh(geometry, material),
);
visuals.scenes.push(scene, scene);
const parent = new THREE.Group();
parent.add(visuals.root);
visuals.dispose();
visuals.dispose();
assert.deepEqual(counts, { geometry: 1, material: 1, texture: 1 });
assert.equal(closed, 1);
assert.equal(parent.children.length, 0);
});
test("missing body bindings fail before any asset request", async () => {
const mujoco = { mj_name2id: () => -1, mjtObj: { mjOBJ_BODY: { value: 1 } } };
await assert.rejects(
BodyVisuals.load(
mujoco,
{},
{},
{
config,
asset: () => {
throw new Error("must not fetch");
},
},
),
/missing robot body trunk/,
);
});
+16
View File
@@ -16,6 +16,8 @@
- `model.xml` 与它实际声明的 25 个 STL;原模型不修改,运行时组合空地与 timestep。
- `policies/walk.onnx`。
- `visuals/*.glb`:用户提供的新版 `microban2.zip` 中 20 个原始 GLB,保留 PBR 材质和 46 张内嵌纹理(45 张法线贴图及 1 张屏幕颜色贴图);替代白模显示,不替换物理网格。
- `visual-source-manifest.json`:视觉压缩包和逐文件 SHA-256,与原物理 / 策略来源记录分开。压缩包没有许可说明,公开分发前需向资产所有者确认。
- `source-manifest.json`:源 remote、HEAD、dirty 标志、逐文件 SHA-256、行为参考代码哈希。
- `SOURCE.md` 和 `LICENSES/`:原项目混合许可文本、XML 的 Apache-2.0 许可文本和归属说明。
@@ -28,6 +30,20 @@ npm --prefix app run validate:packages
导入是显式离线维护操作,不是运行或构建的依赖;日常运行不读取源仓库、不要求 Python / BAM / Placo。导入后需检查是否有实质变化,不自动接受策略 / 行为更新。
## 带纹理外观
刷新页面后选择 Microban 即使用新外观:浅紫外壳、蓝色金属支架、黑色电机及带颜色贴图的屏幕。新版 20 个 GLB 的顶点位置和节点变换与上一版一致,保留现有部件绑定。`render.body_visuals` 将 BASE、LINK1–19 映射到已有的 20 个 MuJoCo body,显式记录米制平移和 xyzw 单位四元数。坐标变换经原几何对齐检查;部件包围盒端点差异最大约 3 mm,包含新外观的螺丝等细节,不是运行时自动猜测比例或改变关节姿态。
所有部件每帧读取原 `xpos/xquat`,包括独立转头;原 visual geom 隐藏,碰撞、质量、惯量、关节、策略和 PD 均保留。GLB 加载失败会放弃候选机器人并恢复旧机器人,不会以静默白模掩盖缺失资源。切换时释放几何、材质、纹理和 ImageBitmap。
```bash
python3 app/scripts/import-microban-visuals.py /path/to/microban2.zip
npm --prefix app run validate:packages
npm --prefix app run test:e2e -- microban-visuals.spec.js microban-walking.spec.js runtime-leaks.spec.js
```
`microban-visuals.spec.js` 验证 20 个 body 的材质/纹理、原 visual 隐藏、非零根 yaw 和关节角下的对齐、失败回退,以及开关新视觉后 80 个控制周期的 qpos / qvel / ONNX action 逐项完全一致。截图保存于该测试的 `test-results/` 输出目录。行走原有三轮有界 70 s、三轮无界诊断 84 s 和 H / reset / 近 50 Hz 验证通过;20 次跨 Microduck / Microban / RP1 切换后,Microban 的 GPU 纹理数稳定为 48(46 张模型纹理加场景资源),无增长。
## 策略接口
| 项目 | 数值 |
+21
View File
@@ -41,6 +41,27 @@ app/public/robots/example/
- `app/src/platform/loader/world.schema.json`
- `SchemaValidator.js` 的跨字段检查(所有权、输入绑定、命令尺寸等)。
## 可选分部件 GLB 外观
通用 runtime 保留 `render.source: "mjcf_visual"`,可增加 `render.body_visuals`,仅替换指定 body 的可见几何:
```json
"body_visuals": [
{
"body": "trunk",
"path": "visuals/trunk.glb",
"position": [0, 0, 0],
"quaternion": [0, 0, 0, 1]
}
]
```
`body` 必须是 MJCF 中存在的非 world body,且不能重复。`position` 为米制 body-local 平移;`quaternion` 为 xyzw 单位四元数;省略时分别使用零平移 / 单位旋转。不按导出文件名猜 body,不使用 MuJoCo 编译器重心校正后的 geom 坐标绑定 GLB。模型单位须预先为米,不支持隐式自动缩放。
GLB 必须自包含(纹理/二进制内嵌),不能引用外部图片或 buffer;`path` 是包内相对 `.glb` 路径。资源经 PackageLoader 加载,保留部署前缀与同源签名。GLB 不加入 `assets` / VFS,因为它们不参与物理编译。隐藏原 visual 不修改原碰撞、惯量、关节、策略或动作。缺失 body 或资源导致候选加载失败,并通过现有切换流程回退。
新增视觉资源应单独保存 `visual-source-manifest.json`,包含 `files: [{"file":"visuals/trunk.glb","sha256":"..."}]`;`validate:packages` 同时验证原来源记录、视觉记录和 GLB 自包含约束。示例与实测记录见 [Microban](microban.md)。
## 场地与机器人尺寸
在 Robot Package 顶层声明参考站姿的全身高度(米),例如: