Files
chenlin c32df70326
Checks / Layout, translation and Colab checks (push) Has been cancelled
Checks / Ruff (push) Has been cancelled
Checks / Framework tests (native backend, CPU) (push) Has been cancelled
Checks / Portable Rust controller (push) Has been cancelled
release: v0.1 generic robot training and application platform
2026-11-05 11:05:01 +08:00

377 lines
16 KiB
Python

"""Invariants of the native camera sensor, and of the robot's own camera.
Two subjects, both about cameras. The first and larger half is mjrl's offscreen
renderer; the robot's forward camera is at the bottom, under its own banner.
Every one of these comes from a side-by-side run on the training machine on
2026-09-03, and each corresponds to a bug that **raised nothing, produced no NaN
and gave a depth image that looked entirely normal**. Static review finds none of
them; only an assertion pins them down.
The comparison criteria are not here: they need mjwarp running as the reference
(see verify_camera_pixels.py / verify_camera_env.py in the scratchpad). What is
pinned here is the behaviour those comparisons have **already** validated and
which must not drift back.
"""
from __future__ import annotations
import pytest
mujoco = pytest.importorskip("mujoco", reason="the camera is MuJoCo's offscreen rendering")
import numpy as np # noqa: E402
from mjrl.sensor.camera import _CATMASK, _no_multisampling, scene_option # noqa: E402
XML_TEXTURED = """
<mujoco>
<asset>
<texture name="grid" type="2d" builtin="checker" rgb1="0.9 0.9 0.9"
rgb2="0.1 0.1 0.1" width="32" height="32"/>
<material name="gridmat" texture="grid" texrepeat="4 4"/>
</asset>
<worldbody>
<geom name="floor" type="plane" size="5 5 0.1" material="gridmat"/>
<camera name="cam" pos="0 0 3" quat="1 0 0 0" fovy="45" resolution="8 8"/>
</worldbody>
</mujoco>
"""
XML = """
<mujoco>
<statistic extent="2"/>
<visual><global offwidth="640" offheight="480"/></visual>
<worldbody>
<light pos="0 0 3"/>
<geom name="floor" type="plane" size="5 5 0.1"/>
<body name="b" pos="0 0 0.4"><freejoint/>
<geom name="box" type="box" size="0.2 0.2 0.2"/>
</body>
<site name="s" pos="0.3 0 0.9" size="0.15"/>
<camera name="cam" pos="0 0 3" quat="1 0 0 0" fovy="45" resolution="32 24"/>
</worldbody>
</mujoco>
"""
@pytest.fixture(scope="module")
def model_data():
m = mujoco.MjModel.from_xml_string(XML)
d = mujoco.MjData(m)
mujoco.mj_forward(m, d)
return m, d
# ── Upstream behaviour we rely on ───────────────────────────────────────
def test_mj_setconst_clobbers_the_visual_statistics(model_data) -> None:
"""`mj_setConst` recomputes an explicitly set `stat.extent` from the geometry.
The restore logic in `NativeSimulation._scatter_model` exists for exactly
that. This test pins **the upstream behaviour**: the day MuJoCo stops
recomputing stat, that restore becomes dead code and this test fails first,
prompting its removal rather than leaving it lying around.
"""
m, d = model_data
m2 = mujoco.MjModel.from_xml_string(XML)
assert m2.stat.extent == pytest.approx(2.0), "the extent set explicitly in the XML"
mujoco.mj_setConst(m2, mujoco.MjData(m2))
assert m2.stat.extent != pytest.approx(2.0), (
"mj_setConst no longer recomputes stat.extent -- "
"the restore logic in NativeSimulation._scatter_model can go"
)
def test_multisampling_is_what_breaks_depth(model_data) -> None:
"""With MSAA on, depth is systematically biased; with it off, it is not.
Measured: on NVIDIA + EGL, MSAA's depth resolve introduces a constant offset
in **inverse depth** -- about 1.5% at 3 m, 4.6% at 10 m -- and it varies with
the sample count. It is invisible on macOS's GL stack (MuJoCo bypasses that
path in split mode), so validating on one machine only would miss it.
This does not compare absolute accuracy (that depends on the GL stack), only
whether turning it off gets **closer to the analytic truth** -- a relation
that holds on both stacks.
"""
m, _ = model_data
d = mujoco.MjData(m)
d.qpos[:3] = [0.05, -0.05, 0.4]
mujoco.mj_forward(m, d)
cam_id = mujoco.mj_name2id(m, mujoco.mjtObj.mjOBJ_CAMERA, "cam")
truth = _analytic_depth(m, d, cam_id, 32, 24)
errs = {}
for label, ctx in (("MSAA off", _no_multisampling(m)), ("MSAA on", _identity())):
try:
with ctx:
r = mujoco.Renderer(m, height=24, width=32)
except Exception as e: # noqa: BLE001
pytest.skip(f"cannot create an offscreen context here: {type(e).__name__}: {e}")
try:
_update(r, m, d, cam_id)
r.enable_depth_rendering()
z = r.render().copy()
r.disable_depth_rendering()
finally:
try:
r.close()
except Exception: # noqa: BLE001,S110
pass
ok = (truth > 0) & (z > 0) & (z < float(m.vis.map.zfar * m.stat.extent) * 0.9)
assert ok.sum() > 50, f"{label}: too few comparable pixels"
errs[label] = float(np.median(np.abs(z[ok] - truth[ok]) / truth[ok]))
assert errs["MSAA off"] <= errs["MSAA on"], (
f"turning MSAA off made it worse: {errs} -- the premise of "
f"_no_multisampling has changed"
)
assert errs["MSAA off"] < 1e-3, (
f"still off the analytic truth with MSAA off: {errs['MSAA off']:.3e}"
)
# ── Our own conventions ─────────────────────────────────────────────────
def test_scene_option_excludes_sites(model_data) -> None:
"""Sites must be excluded.
mjwarp casts geoms only; `mjv_updateScene` draws sites as well by default. A
single site in the scene conjures up a patch in native's depth image that
warp does not have -- and it looks perfectly normal. A site is **not**
mjCAT_DECOR, so the catmask does not stop it; only zeroing sitegroup does.
"""
opt = scene_option((0, 1, 2))
assert list(opt.sitegroup) == [0] * len(opt.sitegroup)
assert list(opt.geomgroup[:3]) == [1, 1, 1]
assert list(opt.geomgroup[3:]) == [0] * (len(opt.geomgroup) - 3)
m, d = model_data
scn = mujoco.MjvScene(model=m, maxgeom=1000)
cam = mujoco.MjvCamera()
cam.type = mujoco.mjtCamera.mjCAMERA_FIXED
cam.fixedcamid = mujoco.mj_name2id(m, mujoco.mjtObj.mjOBJ_CAMERA, "cam")
mujoco.mjv_updateScene(m, d, opt, None, cam, _CATMASK, scn)
types = {int(scn.geoms[i].objtype) for i in range(scn.ngeom)}
assert int(mujoco.mjtObj.mjOBJ_SITE) not in types, f"a site got in: {types}"
# Control: with the defaults, sites really are drawn -- otherwise the
# assertion above proves nothing.
scn2 = mujoco.MjvScene(model=m, maxgeom=1000)
mujoco.mjv_updateScene(m, d, mujoco.MjvOption(), None, cam,
int(mujoco.mjtCatBit.mjCAT_ALL), scn2)
types2 = {int(scn2.geoms[i].objtype) for i in range(scn2.ngeom)}
assert int(mujoco.mjtObj.mjOBJ_SITE) in types2, (
"the control group drew no site either, so this test is vacuous"
)
def test_textures_are_disabled_via_matid_not_texid(model_data) -> None:
"""`use_textures=False` works by setting `matid = -1`; `texid` does nothing.
Measured: setting `texid = -1` left the floor's brightness std at 79.10 ->
79.10 (no change at all); `matid = -1` genuinely removed the checkerboard
(79.10 -> 34.89, matching the 35.20 mjwarp gives for `use_textures=False`).
Colour does not go with it -- `mjv_updateScene` has already resolved the
material's rgba into `geom.rgba`.
This pins down *which field is the switch*. Getting it wrong raises nothing:
the texture renders as before while the config says it is off -- another bug
that looks entirely normal.
"""
src = (
pytest.importorskip("mjrl.sensor.camera").__file__
)
body = open(src, encoding="utf-8").read()
render_one = body.split("def _render_one", 1)[1].split("\n def ", 1)[0]
assert ".matid = -1" in render_one, "the texture switch has to be matid"
assert ".texid = -1" not in render_one, "texid does nothing; this was measured"
def test_textured_rgb_divergence_is_announced(model_data) -> None:
"""RGB plus textures makes the two backends differ, so it must warn.
Measured on example_robot: everything untextured differs by 0.1/255, textured terrain
by 50/255. For a vision policy the texture is most of the picture -- leave it
unsaid and it becomes a problem discovered three days into training.
Only RGB is affected; depth and segmentation have nothing to do with
textures and should not be bothered by this.
"""
from mjrl.sensor.camera import setup_context
m = mujoco.MjModel.from_xml_string(XML_TEXTURED)
def ctx_with(types, use_textures=True):
cfg = type("C", (), dict(name="c", width=8, height=8, data_types=types,
enabled_geom_groups=(0, 1, 2),
use_textures=use_textures, use_shadows=False))()
c = type("Ctx", (), {})()
c.camera_sensors = [type("S", (), dict(cfg=cfg, camera_idx=0))()]
c._data = type("D", (), {"nworld": 1})()
return c
with pytest.warns(RuntimeWarning, match="textured materials"):
setup_context(ctx_with(("rgb",)), m)
import warnings as _w
for types, tex in ((("depth", "segmentation"), True), (("rgb",), False)):
with _w.catch_warnings():
_w.simplefilter("error") # any warning at all counts as a failure
setup_context(ctx_with(types, tex), m)
def test_no_multisampling_restores_on_exception(model_data) -> None:
"""offsamples must be restored on the exception path too -- it mutates the
shared model."""
m, _ = model_data
before = int(m.vis.quality.offsamples)
with pytest.raises(RuntimeError):
with _no_multisampling(m):
assert int(m.vis.quality.offsamples) == 0
raise RuntimeError("boom")
assert int(m.vis.quality.offsamples) == before
def test_setup_context_lays_out_buffers_like_mjwarp() -> None:
"""With several cameras, **each data type gets its own running offset**, and
a camera that does not enable a type records -1.
A wrong offset raises nothing: `get_depth()` still returns a
correctly-shaped tensor, it just holds another camera's content, or reads
out of bounds and gets zeros. Hence a deliberately different camera
combination for each of the three data types.
"""
from mjrl.sensor.camera import setup_context
def sensor(name, w, h, types, idx):
cfg = type("C", (), dict(name=name, width=w, height=h, data_types=types,
enabled_geom_groups=(0, 1, 2), use_textures=True,
use_shadows=False))()
return type("S", (), dict(cfg=cfg, camera_idx=idx))()
ctx = type("Ctx", (), {})()
ctx.camera_sensors = [
sensor("a", 8, 6, ("rgb", "depth", "segmentation"), 0),
sensor("b", 4, 4, ("depth",), 1),
sensor("c", 5, 5, ("segmentation", "rgb"), 2),
]
ctx._data = type("D", (), {"nworld": 3})()
setup_context(ctx, mujoco.MjModel.from_xml_string(XML))
assert ctx._depth_adr_np == [0, 48, -1], (
"only cameras that asked for depth take space in the depth buffer"
)
assert ctx._seg_adr_np == [0, -1, 48], "seg has its own independent offsets"
assert ctx._rgb_adr_np == [0, -1, 48], "and rgb another set again"
assert tuple(ctx._depth_torch.shape) == (3, 48 + 16)
assert tuple(ctx._seg_torch.shape) == (3, 48 + 25, 2)
assert tuple(ctx._rgb_torch.shape) == (3, 48 + 25, 3)
assert ctx._rgb_torch.dtype.__str__() == "torch.uint8", "RGB has to be uint8"
# get_rgb() uses _rgb_unpacked to decide "is RGB on"; on native it is the
# very same buffer.
assert ctx._rgb_unpacked is ctx._rgb_torch
# Segmentation's background convention is -1, not 0 -- the initial value has
# to be right, since it can be read before the first frame is rendered.
assert int(ctx._seg_torch.min()) == -1 and int(ctx._seg_torch.max()) == -1
def test_setup_context_skips_buffers_nobody_asked_for() -> None:
"""With no camera asking for rgb, `_rgb_unpacked` must be None.
That is exactly what `get_rgb()` uses to decide "is RGB on". Allocating
unconditionally would hand a misconfigured task an image that is forever
black instead of an error -- again the kind that looks plausible.
"""
from mjrl.sensor.camera import setup_context
cfg = type("C", (), dict(name="d", width=4, height=4, data_types=("depth",),
enabled_geom_groups=(0, 1, 2), use_textures=True,
use_shadows=False))()
ctx = type("Ctx", (), {})()
ctx.camera_sensors = [type("S", (), dict(cfg=cfg, camera_idx=0))()]
ctx._data = type("D", (), {"nworld": 2})()
setup_context(ctx, mujoco.MjModel.from_xml_string(XML))
assert ctx._rgb_torch is None and ctx._rgb_unpacked is None
assert ctx._seg_torch is None
assert ctx._rgb_adr_np == [-1] and ctx._seg_adr_np == [-1]
def test_render_flags_match_what_mjwarp_can_do() -> None:
"""Rasteriser-only effects must be off: reflection, fog and haze do not
exist in mjwarp at all.
RGB can never match pixel for pixel across the two, but *whether a class of
effect is drawn* can be aligned. Left on, floor reflection and fog shift the
colour of everything far away, and that part of the difference **has nothing
to do with the shading model** -- mixed together, there is no way left to say
what causes the rest.
"""
from mjrl.sensor.camera import _align_render_flags
m = mujoco.MjModel.from_xml_string(XML)
scn = mujoco.MjvScene(model=m, maxgeom=100)
f = mujoco.mjtRndFlag
for flag in (f.mjRND_REFLECTION, f.mjRND_FOG, f.mjRND_HAZE):
scn.flags[flag] = True # on first, to confirm they get turned off
_align_render_flags(scn, use_shadows=False)
assert not scn.flags[f.mjRND_REFLECTION]
assert not scn.flags[f.mjRND_FOG]
assert not scn.flags[f.mjRND_HAZE]
assert not scn.flags[f.mjRND_SHADOW]
assert scn.flags[f.mjRND_SKYBOX], "mjlab passes render_skybox=True to mjwarp"
_align_render_flags(scn, use_shadows=True)
assert scn.flags[f.mjRND_SHADOW], "use_shadows=True must actually enable shadows"
# ── Helpers ─────────────────────────────────────────────────────────────
import contextlib # noqa: E402
@contextlib.contextmanager
def _identity():
yield
def _update(r, m, d, cam_id) -> None:
cam = mujoco.MjvCamera()
cam.type = mujoco.mjtCamera.mjCAMERA_FIXED
cam.fixedcamid = cam_id
mujoco.mjv_updateScene(m, d, scene_option((0, 1, 2)), None, cam, _CATMASK, r.scene)
def _analytic_depth(m, d, cam_id, w, h) -> np.ndarray:
"""Solve the analytic plane depth per pixel with `mujoco.mj_ray`, as a
reference independent of rasterisation.
Ray directions are built the way mjwarp's `compute_ray` does, and depth is
the projection along the optical axis -- matching both backends' convention
(see the comparison table at the top of mjrl/sensor/camera.py).
"""
znear = float(m.vis.map.znear * m.stat.extent)
half_h = znear * np.tan(np.radians(float(m.cam_fovy[cam_id]) / 2))
half_w = half_h * (w / h)
pos = d.cam_xpos[cam_id].copy()
mat = d.cam_xmat[cam_id].reshape(3, 3).copy()
out = np.zeros((h, w))
gid = np.zeros(1, np.int32)
for py in range(h):
y = half_h - 2 * half_h * ((py + 0.5) / h)
for px in range(w):
x = -half_w + 2 * half_w * ((px + 0.5) / w)
v = np.array([x, y, -znear])
v /= np.linalg.norm(v)
dist = mujoco.mj_ray(m, d, pos, mat @ v, None, 1, -1, gid)
out[py, px] = dist * (-v[2]) if dist >= 0 else 0.0
return out