"""Invariants of the native camera sensor, and of the robot's own camera. Two subjects, both about cameras. The first and larger half is mjrl's offscreen renderer; the robot's forward camera is at the bottom, under its own banner. Every one of these comes from a side-by-side run on the training machine on 2026-09-03, and each corresponds to a bug that **raised nothing, produced no NaN and gave a depth image that looked entirely normal**. Static review finds none of them; only an assertion pins them down. The comparison criteria are not here: they need mjwarp running as the reference (see verify_camera_pixels.py / verify_camera_env.py in the scratchpad). What is pinned here is the behaviour those comparisons have **already** validated and which must not drift back. """ from __future__ import annotations import pytest mujoco = pytest.importorskip("mujoco", reason="the camera is MuJoCo's offscreen rendering") import numpy as np # noqa: E402 from mjrl.sensor.camera import _CATMASK, _no_multisampling, scene_option # noqa: E402 XML_TEXTURED = """ """ XML = """ """ @pytest.fixture(scope="module") def model_data(): m = mujoco.MjModel.from_xml_string(XML) d = mujoco.MjData(m) mujoco.mj_forward(m, d) return m, d # ── Upstream behaviour we rely on ─────────────────────────────────────── def test_mj_setconst_clobbers_the_visual_statistics(model_data) -> None: """`mj_setConst` recomputes an explicitly set `stat.extent` from the geometry. The restore logic in `NativeSimulation._scatter_model` exists for exactly that. This test pins **the upstream behaviour**: the day MuJoCo stops recomputing stat, that restore becomes dead code and this test fails first, prompting its removal rather than leaving it lying around. """ m, d = model_data m2 = mujoco.MjModel.from_xml_string(XML) assert m2.stat.extent == pytest.approx(2.0), "the extent set explicitly in the XML" mujoco.mj_setConst(m2, mujoco.MjData(m2)) assert m2.stat.extent != pytest.approx(2.0), ( "mj_setConst no longer recomputes stat.extent -- " "the restore logic in NativeSimulation._scatter_model can go" ) def test_multisampling_is_what_breaks_depth(model_data) -> None: """With MSAA on, depth is systematically biased; with it off, it is not. Measured: on NVIDIA + EGL, MSAA's depth resolve introduces a constant offset in **inverse depth** -- about 1.5% at 3 m, 4.6% at 10 m -- and it varies with the sample count. It is invisible on macOS's GL stack (MuJoCo bypasses that path in split mode), so validating on one machine only would miss it. This does not compare absolute accuracy (that depends on the GL stack), only whether turning it off gets **closer to the analytic truth** -- a relation that holds on both stacks. """ m, _ = model_data d = mujoco.MjData(m) d.qpos[:3] = [0.05, -0.05, 0.4] mujoco.mj_forward(m, d) cam_id = mujoco.mj_name2id(m, mujoco.mjtObj.mjOBJ_CAMERA, "cam") truth = _analytic_depth(m, d, cam_id, 32, 24) errs = {} for label, ctx in (("MSAA off", _no_multisampling(m)), ("MSAA on", _identity())): try: with ctx: r = mujoco.Renderer(m, height=24, width=32) except Exception as e: # noqa: BLE001 pytest.skip(f"cannot create an offscreen context here: {type(e).__name__}: {e}") try: _update(r, m, d, cam_id) r.enable_depth_rendering() z = r.render().copy() r.disable_depth_rendering() finally: try: r.close() except Exception: # noqa: BLE001,S110 pass ok = (truth > 0) & (z > 0) & (z < float(m.vis.map.zfar * m.stat.extent) * 0.9) assert ok.sum() > 50, f"{label}: too few comparable pixels" errs[label] = float(np.median(np.abs(z[ok] - truth[ok]) / truth[ok])) assert errs["MSAA off"] <= errs["MSAA on"], ( f"turning MSAA off made it worse: {errs} -- the premise of " f"_no_multisampling has changed" ) assert errs["MSAA off"] < 1e-3, ( f"still off the analytic truth with MSAA off: {errs['MSAA off']:.3e}" ) # ── Our own conventions ───────────────────────────────────────────────── def test_scene_option_excludes_sites(model_data) -> None: """Sites must be excluded. mjwarp casts geoms only; `mjv_updateScene` draws sites as well by default. A single site in the scene conjures up a patch in native's depth image that warp does not have -- and it looks perfectly normal. A site is **not** mjCAT_DECOR, so the catmask does not stop it; only zeroing sitegroup does. """ opt = scene_option((0, 1, 2)) assert list(opt.sitegroup) == [0] * len(opt.sitegroup) assert list(opt.geomgroup[:3]) == [1, 1, 1] assert list(opt.geomgroup[3:]) == [0] * (len(opt.geomgroup) - 3) m, d = model_data scn = mujoco.MjvScene(model=m, maxgeom=1000) cam = mujoco.MjvCamera() cam.type = mujoco.mjtCamera.mjCAMERA_FIXED cam.fixedcamid = mujoco.mj_name2id(m, mujoco.mjtObj.mjOBJ_CAMERA, "cam") mujoco.mjv_updateScene(m, d, opt, None, cam, _CATMASK, scn) types = {int(scn.geoms[i].objtype) for i in range(scn.ngeom)} assert int(mujoco.mjtObj.mjOBJ_SITE) not in types, f"a site got in: {types}" # Control: with the defaults, sites really are drawn -- otherwise the # assertion above proves nothing. scn2 = mujoco.MjvScene(model=m, maxgeom=1000) mujoco.mjv_updateScene(m, d, mujoco.MjvOption(), None, cam, int(mujoco.mjtCatBit.mjCAT_ALL), scn2) types2 = {int(scn2.geoms[i].objtype) for i in range(scn2.ngeom)} assert int(mujoco.mjtObj.mjOBJ_SITE) in types2, ( "the control group drew no site either, so this test is vacuous" ) def test_textures_are_disabled_via_matid_not_texid(model_data) -> None: """`use_textures=False` works by setting `matid = -1`; `texid` does nothing. Measured: setting `texid = -1` left the floor's brightness std at 79.10 -> 79.10 (no change at all); `matid = -1` genuinely removed the checkerboard (79.10 -> 34.89, matching the 35.20 mjwarp gives for `use_textures=False`). Colour does not go with it -- `mjv_updateScene` has already resolved the material's rgba into `geom.rgba`. This pins down *which field is the switch*. Getting it wrong raises nothing: the texture renders as before while the config says it is off -- another bug that looks entirely normal. """ src = ( pytest.importorskip("mjrl.sensor.camera").__file__ ) body = open(src, encoding="utf-8").read() render_one = body.split("def _render_one", 1)[1].split("\n def ", 1)[0] assert ".matid = -1" in render_one, "the texture switch has to be matid" assert ".texid = -1" not in render_one, "texid does nothing; this was measured" def test_textured_rgb_divergence_is_announced(model_data) -> None: """RGB plus textures makes the two backends differ, so it must warn. Measured on example_robot: everything untextured differs by 0.1/255, textured terrain by 50/255. For a vision policy the texture is most of the picture -- leave it unsaid and it becomes a problem discovered three days into training. Only RGB is affected; depth and segmentation have nothing to do with textures and should not be bothered by this. """ from mjrl.sensor.camera import setup_context m = mujoco.MjModel.from_xml_string(XML_TEXTURED) def ctx_with(types, use_textures=True): cfg = type("C", (), dict(name="c", width=8, height=8, data_types=types, enabled_geom_groups=(0, 1, 2), use_textures=use_textures, use_shadows=False))() c = type("Ctx", (), {})() c.camera_sensors = [type("S", (), dict(cfg=cfg, camera_idx=0))()] c._data = type("D", (), {"nworld": 1})() return c with pytest.warns(RuntimeWarning, match="textured materials"): setup_context(ctx_with(("rgb",)), m) import warnings as _w for types, tex in ((("depth", "segmentation"), True), (("rgb",), False)): with _w.catch_warnings(): _w.simplefilter("error") # any warning at all counts as a failure setup_context(ctx_with(types, tex), m) def test_no_multisampling_restores_on_exception(model_data) -> None: """offsamples must be restored on the exception path too -- it mutates the shared model.""" m, _ = model_data before = int(m.vis.quality.offsamples) with pytest.raises(RuntimeError): with _no_multisampling(m): assert int(m.vis.quality.offsamples) == 0 raise RuntimeError("boom") assert int(m.vis.quality.offsamples) == before def test_setup_context_lays_out_buffers_like_mjwarp() -> None: """With several cameras, **each data type gets its own running offset**, and a camera that does not enable a type records -1. A wrong offset raises nothing: `get_depth()` still returns a correctly-shaped tensor, it just holds another camera's content, or reads out of bounds and gets zeros. Hence a deliberately different camera combination for each of the three data types. """ from mjrl.sensor.camera import setup_context def sensor(name, w, h, types, idx): cfg = type("C", (), dict(name=name, width=w, height=h, data_types=types, enabled_geom_groups=(0, 1, 2), use_textures=True, use_shadows=False))() return type("S", (), dict(cfg=cfg, camera_idx=idx))() ctx = type("Ctx", (), {})() ctx.camera_sensors = [ sensor("a", 8, 6, ("rgb", "depth", "segmentation"), 0), sensor("b", 4, 4, ("depth",), 1), sensor("c", 5, 5, ("segmentation", "rgb"), 2), ] ctx._data = type("D", (), {"nworld": 3})() setup_context(ctx, mujoco.MjModel.from_xml_string(XML)) assert ctx._depth_adr_np == [0, 48, -1], ( "only cameras that asked for depth take space in the depth buffer" ) assert ctx._seg_adr_np == [0, -1, 48], "seg has its own independent offsets" assert ctx._rgb_adr_np == [0, -1, 48], "and rgb another set again" assert tuple(ctx._depth_torch.shape) == (3, 48 + 16) assert tuple(ctx._seg_torch.shape) == (3, 48 + 25, 2) assert tuple(ctx._rgb_torch.shape) == (3, 48 + 25, 3) assert ctx._rgb_torch.dtype.__str__() == "torch.uint8", "RGB has to be uint8" # get_rgb() uses _rgb_unpacked to decide "is RGB on"; on native it is the # very same buffer. assert ctx._rgb_unpacked is ctx._rgb_torch # Segmentation's background convention is -1, not 0 -- the initial value has # to be right, since it can be read before the first frame is rendered. assert int(ctx._seg_torch.min()) == -1 and int(ctx._seg_torch.max()) == -1 def test_setup_context_skips_buffers_nobody_asked_for() -> None: """With no camera asking for rgb, `_rgb_unpacked` must be None. That is exactly what `get_rgb()` uses to decide "is RGB on". Allocating unconditionally would hand a misconfigured task an image that is forever black instead of an error -- again the kind that looks plausible. """ from mjrl.sensor.camera import setup_context cfg = type("C", (), dict(name="d", width=4, height=4, data_types=("depth",), enabled_geom_groups=(0, 1, 2), use_textures=True, use_shadows=False))() ctx = type("Ctx", (), {})() ctx.camera_sensors = [type("S", (), dict(cfg=cfg, camera_idx=0))()] ctx._data = type("D", (), {"nworld": 2})() setup_context(ctx, mujoco.MjModel.from_xml_string(XML)) assert ctx._rgb_torch is None and ctx._rgb_unpacked is None assert ctx._seg_torch is None assert ctx._rgb_adr_np == [-1] and ctx._seg_adr_np == [-1] def test_render_flags_match_what_mjwarp_can_do() -> None: """Rasteriser-only effects must be off: reflection, fog and haze do not exist in mjwarp at all. RGB can never match pixel for pixel across the two, but *whether a class of effect is drawn* can be aligned. Left on, floor reflection and fog shift the colour of everything far away, and that part of the difference **has nothing to do with the shading model** -- mixed together, there is no way left to say what causes the rest. """ from mjrl.sensor.camera import _align_render_flags m = mujoco.MjModel.from_xml_string(XML) scn = mujoco.MjvScene(model=m, maxgeom=100) f = mujoco.mjtRndFlag for flag in (f.mjRND_REFLECTION, f.mjRND_FOG, f.mjRND_HAZE): scn.flags[flag] = True # on first, to confirm they get turned off _align_render_flags(scn, use_shadows=False) assert not scn.flags[f.mjRND_REFLECTION] assert not scn.flags[f.mjRND_FOG] assert not scn.flags[f.mjRND_HAZE] assert not scn.flags[f.mjRND_SHADOW] assert scn.flags[f.mjRND_SKYBOX], "mjlab passes render_skybox=True to mjwarp" _align_render_flags(scn, use_shadows=True) assert scn.flags[f.mjRND_SHADOW], "use_shadows=True must actually enable shadows" # ── Helpers ───────────────────────────────────────────────────────────── import contextlib # noqa: E402 @contextlib.contextmanager def _identity(): yield def _update(r, m, d, cam_id) -> None: cam = mujoco.MjvCamera() cam.type = mujoco.mjtCamera.mjCAMERA_FIXED cam.fixedcamid = cam_id mujoco.mjv_updateScene(m, d, scene_option((0, 1, 2)), None, cam, _CATMASK, r.scene) def _analytic_depth(m, d, cam_id, w, h) -> np.ndarray: """Solve the analytic plane depth per pixel with `mujoco.mj_ray`, as a reference independent of rasterisation. Ray directions are built the way mjwarp's `compute_ray` does, and depth is the projection along the optical axis -- matching both backends' convention (see the comparison table at the top of mjrl/sensor/camera.py). """ znear = float(m.vis.map.znear * m.stat.extent) half_h = znear * np.tan(np.radians(float(m.cam_fovy[cam_id]) / 2)) half_w = half_h * (w / h) pos = d.cam_xpos[cam_id].copy() mat = d.cam_xmat[cam_id].reshape(3, 3).copy() out = np.zeros((h, w)) gid = np.zeros(1, np.int32) for py in range(h): y = half_h - 2 * half_h * ((py + 0.5) / h) for px in range(w): x = -half_w + 2 * half_w * ((px + 0.5) / w) v = np.array([x, y, -znear]) v /= np.linalg.norm(v) dist = mujoco.mj_ray(m, d, pos, mat @ v, None, 1, -1, gid) out[py, px] = dist * (-v[2]) if dist >= 0 else 0.0 return out