feat(sim): filament renderer integration

docker/Dockerfile

@@ -48,7 +48,7 @@ RUN chmod +x /usr/local/bin/link-editable-source \
     pybind11 \
     cmake \
     ninja \
-    "mujoco==3.2.6" \
+    "mujoco==3.15.0" \
     "pin==3.7.0" \
     && uv pip install /opt/rcs-src \
     && uv pip install --no-build-isolation /opt/rcs-src/extensions/rcs_fr3 \

docs/user_guide/scene_configuration.md

@@ -299,6 +299,36 @@ Example:
 
 - `depth[y, x] == 1500` means the point is about **1.5 m** away from the camera
 
+## Rendering backend (Filament)
+
+By default, simulation cameras and the GUI use MuJoCo's classic OpenGL renderer. MuJoCo >= 3.15 also
+ships the physically based [Filament](https://github.com/google/filament) renderer and the new
+MuJoCo Studio viewer. Switch to them with the `renderer` flag of `SimConfig`:
+
+```python
+from rcs.sim import RendererBackend, SimConfig
+
+sim_cfg = SimConfig(async_control=False, realtime=False, renderer=RendererBackend.FILAMENT)
+```
+
+With this flag
+
+- `SimCameraSet(...)` returns a `FilamentSimCameraSet`: color images are rendered with Filament in
+  Python (frames are rendered when requested, a fixed camera frame rate is not supported). Filament does
+  not expose a metric depth buffer through MuJoCo's Python API, so depth images are still rendered with
+  the classic renderer and always use the metric units described above. The depth pass costs about as
+  much as the whole classic renderer, so it is skipped when `CameraSetWrapper(include_depth=False)`
+  (or `FilamentSimCameraSet(render_depth=False)`) is used.
+- `open_gui()` launches MuJoCo Studio instead of the passive `mujoco.viewer`. Studio runs in the GUI
+  subprocess under a regular `python` interpreter (not `mjpython`, also on macOS).
+
+Environment variables for fine tuning:
+
+- `RCS_FILAMENT_GRAPHICS_API=opengl|vulkan` graphics API of the camera renderer (default `opengl`)
+- `RCS_FILAMENT_SOFTWARE_RENDERING=1` force software rendering for the cameras
+- `RCS_FILAMENT_GFX=opengl|vulkan|web|...` Studio graphics mode; `web` serves the viewer over http instead
+  of opening a window
+
 ### Camera placement
 
 #### Fixed scene camera

examples/fr3/fr3_env_cartesian_control.py

@@ -2,7 +2,7 @@
 
 import gymnasium as gym
 from rcs._core.common import RobotPlatform
-from rcs._core.sim import SimConfig
+from rcs._core.sim import RendererBackend, SimConfig
 from rcs.camera.sim import SimCameraSet
 from rcs.envs.base import (
     CameraSetWrapper,
@@ -50,6 +50,7 @@ def main():
         sim_cfg = SimConfig(
             realtime=False,
             async_control=False,
+            renderer=RendererBackend.FILAMENT,
         )
 
         mjmodel = scene.create_model(cfg)

pyproject.toml

@@ -7,7 +7,7 @@ requires = [
   "pybind11",
   "cmake",
   "ninja",
-  "mujoco==3.10.0",
+  "mujoco==3.15.0",
   "pin==3.7.0",
 ]
 build-backend = "scikit_build_core.build"
@@ -36,7 +36,7 @@ dependencies = [
   "rpyc~=6.0.2",
   "pyarrow",
   "simplejpeg",
-  "mujoco==3.10.0",
+  "mujoco==3.15.0",
   "pin==3.7.0",
   # pin 3.7.0 currently resolves against older urdfdom/tinyxml SONAMEs at runtime
   "cmeel-urdfdom<5",
@@ -76,7 +76,7 @@ dev = [
   "twine~=6.0",
 ]
 build_deps = [
-  "mujoco==3.10.0",
+  "mujoco==3.15.0",
   "pin==3.7.0",
   "cmeel-urdfdom<5",
   "cmeel-tinyxml2<11",
@@ -93,7 +93,7 @@ environment = { SKBUILD_BUILD_DIR = "/tmp/rcs-build/{wheel_tag}" }
 archs = ["x86_64"]
 manylinux-x86_64-image = "manylinux_2_28"
 before-all = "yum install -y epel-release && yum install -y libXi-devel libXcursor-devel libXinerama-devel libXrandr-devel glfw-devel"
-repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} --exclude libmujoco.so.3.10.0 --exclude libpinocchio_default.so.3.7.0 --exclude libpinocchio_parsers.so.3.7.0"
+repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} --exclude libmujoco.so.3.15.0 --exclude libpinocchio_default.so.3.7.0 --exclude libpinocchio_parsers.so.3.7.0"
 
 [tool.cibuildwheel.macos]
 archs = ["arm64"]

python/rcs/_core/sim.pyi

@@ -15,6 +15,7 @@ __all__: list[str] = [
     "DynamicJointState",
     "FrameSet",
     "GuiClient",
+    "RendererBackend",
     "Sim",
     "SimCameraConfig",
     "SimCameraSet",
@@ -28,6 +29,8 @@ __all__: list[str] = [
     "SimTilburgHand",
     "SimTilburgHandConfig",
     "SimTilburgHandState",
+    "CLASSIC",
+    "FILAMENT",
     "default_free",
     "fixed",
     "free",
@@ -102,6 +105,35 @@ class GuiClient:
     def set_model_and_data(self, arg0: int, arg1: int) -> None: ...
     def sync(self) -> None: ...
 
+class RendererBackend:
+    """
+    Members:
+
+      CLASSIC
+
+      FILAMENT
+    """
+
+    CLASSIC: typing.ClassVar[RendererBackend]  # value = <RendererBackend.CLASSIC: 0>
+    FILAMENT: typing.ClassVar[RendererBackend]  # value = <RendererBackend.FILAMENT: 1>
+    __members__: typing.ClassVar[
+        dict[str, RendererBackend]
+    ]  # value = {'CLASSIC': <RendererBackend.CLASSIC: 0>, 'FILAMENT': <RendererBackend.FILAMENT: 1>}
+    def __eq__(self, other: typing.Any) -> bool: ...
+    def __getstate__(self) -> int: ...
+    def __hash__(self) -> int: ...
+    def __index__(self) -> int: ...
+    def __init__(self, value: int) -> None: ...
+    def __int__(self) -> int: ...
+    def __ne__(self, other: typing.Any) -> bool: ...
+    def __repr__(self) -> str: ...
+    def __setstate__(self, state: int) -> None: ...
+    def __str__(self) -> str: ...
+    @property
+    def name(self) -> str: ...
+    @property
+    def value(self) -> int: ...
+
 class Sim:
     def __init__(self, mjmdl: int, mjdata: int) -> None: ...
     def _start_gui_server(self, id: str) -> None: ...
@@ -141,6 +173,7 @@ class SimConfig:
     frequency: float
     max_convergence_steps: int
     realtime: bool
+    renderer: RendererBackend
     def __copy__(self) -> SimConfig: ...
     def __deepcopy__(self, arg0: dict) -> SimConfig: ...
     def __init__(
@@ -149,6 +182,7 @@ class SimConfig:
         realtime: bool = False,
         frequency: float = 30.0,
         max_convergence_steps: int = 500,
+        renderer: RendererBackend = ...,
     ) -> None: ...
 
 class SimGripper(rcs._core.common.Gripper):
@@ -385,3 +419,5 @@ default_free: CameraType  # value = <CameraType.default_free: 3>
 fixed: CameraType  # value = <CameraType.fixed: 2>
 free: CameraType  # value = <CameraType.free: 0>
 tracking: CameraType  # value = <CameraType.tracking: 1>
+CLASSIC: RendererBackend  # value = <RendererBackend.CLASSIC: 0>
+FILAMENT: RendererBackend  # value = <RendererBackend.FILAMENT: 1>