diff --git a/docker/Dockerfile b/docker/Dockerfile index d40dace2..db9d23c0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -48,7 +48,7 @@ RUN chmod +x /usr/local/bin/link-editable-source \ pybind11 \ cmake \ ninja \ - "mujoco==3.2.6" \ + "mujoco==3.15.0" \ "pin==3.7.0" \ && uv pip install /opt/rcs-src \ && uv pip install --no-build-isolation /opt/rcs-src/extensions/rcs_fr3 \ diff --git a/docs/user_guide/scene_configuration.md b/docs/user_guide/scene_configuration.md index aeddac88..19b8ffc3 100644 --- a/docs/user_guide/scene_configuration.md +++ b/docs/user_guide/scene_configuration.md @@ -299,6 +299,36 @@ Example: - `depth[y, x] == 1500` means the point is about **1.5 m** away from the camera +## Rendering backend (Filament) + +By default, simulation cameras and the GUI use MuJoCo's classic OpenGL renderer. MuJoCo >= 3.15 also +ships the physically based [Filament](https://github.com/google/filament) renderer and the new +MuJoCo Studio viewer. Switch to them with the `renderer` flag of `SimConfig`: + +```python +from rcs.sim import RendererBackend, SimConfig + +sim_cfg = SimConfig(async_control=False, realtime=False, renderer=RendererBackend.FILAMENT) +``` + +With this flag + +- `SimCameraSet(...)` returns a `FilamentSimCameraSet`: color images are rendered with Filament in + Python (frames are rendered when requested, a fixed camera frame rate is not supported). Filament does + not expose a metric depth buffer through MuJoCo's Python API, so depth images are still rendered with + the classic renderer and always use the metric units described above. The depth pass costs about as + much as the whole classic renderer, so it is skipped when `CameraSetWrapper(include_depth=False)` + (or `FilamentSimCameraSet(render_depth=False)`) is used. +- `open_gui()` launches MuJoCo Studio instead of the passive `mujoco.viewer`. Studio runs in the GUI + subprocess under a regular `python` interpreter (not `mjpython`, also on macOS). + +Environment variables for fine tuning: + +- `RCS_FILAMENT_GRAPHICS_API=opengl|vulkan` graphics API of the camera renderer (default `opengl`) +- `RCS_FILAMENT_SOFTWARE_RENDERING=1` force software rendering for the cameras +- `RCS_FILAMENT_GFX=opengl|vulkan|web|...` Studio graphics mode; `web` serves the viewer over http instead + of opening a window + ### Camera placement #### Fixed scene camera diff --git a/examples/fr3/fr3_env_cartesian_control.py b/examples/fr3/fr3_env_cartesian_control.py index ab0b5fea..1cc7fd2b 100644 --- a/examples/fr3/fr3_env_cartesian_control.py +++ b/examples/fr3/fr3_env_cartesian_control.py @@ -2,7 +2,7 @@ import gymnasium as gym from rcs._core.common import RobotPlatform -from rcs._core.sim import SimConfig +from rcs._core.sim import RendererBackend, SimConfig from rcs.camera.sim import SimCameraSet from rcs.envs.base import ( CameraSetWrapper, @@ -50,6 +50,7 @@ def main(): sim_cfg = SimConfig( realtime=False, async_control=False, + renderer=RendererBackend.FILAMENT, ) mjmodel = scene.create_model(cfg) diff --git a/pyproject.toml b/pyproject.toml index 2079ea41..0f0ba562 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires = [ "pybind11", "cmake", "ninja", - "mujoco==3.10.0", + "mujoco==3.15.0", "pin==3.7.0", ] build-backend = "scikit_build_core.build" @@ -36,7 +36,7 @@ dependencies = [ "rpyc~=6.0.2", "pyarrow", "simplejpeg", - "mujoco==3.10.0", + "mujoco==3.15.0", "pin==3.7.0", # pin 3.7.0 currently resolves against older urdfdom/tinyxml SONAMEs at runtime "cmeel-urdfdom<5", @@ -76,7 +76,7 @@ dev = [ "twine~=6.0", ] build_deps = [ - "mujoco==3.10.0", + "mujoco==3.15.0", "pin==3.7.0", "cmeel-urdfdom<5", "cmeel-tinyxml2<11", @@ -93,7 +93,7 @@ environment = { SKBUILD_BUILD_DIR = "/tmp/rcs-build/{wheel_tag}" } archs = ["x86_64"] manylinux-x86_64-image = "manylinux_2_28" before-all = "yum install -y epel-release && yum install -y libXi-devel libXcursor-devel libXinerama-devel libXrandr-devel glfw-devel" -repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} --exclude libmujoco.so.3.10.0 --exclude libpinocchio_default.so.3.7.0 --exclude libpinocchio_parsers.so.3.7.0" +repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} --exclude libmujoco.so.3.15.0 --exclude libpinocchio_default.so.3.7.0 --exclude libpinocchio_parsers.so.3.7.0" [tool.cibuildwheel.macos] archs = ["arm64"] diff --git a/python/rcs/_core/sim.pyi b/python/rcs/_core/sim.pyi index 042e2608..2d67fc7d 100644 --- a/python/rcs/_core/sim.pyi +++ b/python/rcs/_core/sim.pyi @@ -15,6 +15,7 @@ __all__: list[str] = [ "DynamicJointState", "FrameSet", "GuiClient", + "RendererBackend", "Sim", "SimCameraConfig", "SimCameraSet", @@ -28,6 +29,8 @@ __all__: list[str] = [ "SimTilburgHand", "SimTilburgHandConfig", "SimTilburgHandState", + "CLASSIC", + "FILAMENT", "default_free", "fixed", "free", @@ -102,6 +105,35 @@ class GuiClient: def set_model_and_data(self, arg0: int, arg1: int) -> None: ... def sync(self) -> None: ... +class RendererBackend: + """ + Members: + + CLASSIC + + FILAMENT + """ + + CLASSIC: typing.ClassVar[RendererBackend] # value = + FILAMENT: typing.ClassVar[RendererBackend] # value = + __members__: typing.ClassVar[ + dict[str, RendererBackend] + ] # value = {'CLASSIC': , 'FILAMENT': } + def __eq__(self, other: typing.Any) -> bool: ... + def __getstate__(self) -> int: ... + def __hash__(self) -> int: ... + def __index__(self) -> int: ... + def __init__(self, value: int) -> None: ... + def __int__(self) -> int: ... + def __ne__(self, other: typing.Any) -> bool: ... + def __repr__(self) -> str: ... + def __setstate__(self, state: int) -> None: ... + def __str__(self) -> str: ... + @property + def name(self) -> str: ... + @property + def value(self) -> int: ... + class Sim: def __init__(self, mjmdl: int, mjdata: int) -> None: ... def _start_gui_server(self, id: str) -> None: ... @@ -141,6 +173,7 @@ class SimConfig: frequency: float max_convergence_steps: int realtime: bool + renderer: RendererBackend def __copy__(self) -> SimConfig: ... def __deepcopy__(self, arg0: dict) -> SimConfig: ... def __init__( @@ -149,6 +182,7 @@ class SimConfig: realtime: bool = False, frequency: float = 30.0, max_convergence_steps: int = 500, + renderer: RendererBackend = ..., ) -> None: ... class SimGripper(rcs._core.common.Gripper): @@ -385,3 +419,5 @@ default_free: CameraType # value = fixed: CameraType # value = free: CameraType # value = tracking: CameraType # value = +CLASSIC: RendererBackend # value = +FILAMENT: RendererBackend # value = diff --git a/python/rcs/camera/sim.py b/python/rcs/camera/sim.py index 12af75b4..518b4f99 100644 --- a/python/rcs/camera/sim.py +++ b/python/rcs/camera/sim.py @@ -7,20 +7,71 @@ # from rcs._core.common import BaseCameraConfig from rcs._core import common +from rcs._core.sim import CameraType from rcs._core.sim import FrameSet as _FrameSet -from rcs._core.sim import SimCameraConfig +from rcs._core.sim import RendererBackend, SimCameraConfig from rcs._core.sim import SimCameraSet as _SimCameraSet from rcs.camera.interface import BaseCameraSet, CameraFrame, DataFrame, Frame, FrameSet -from rcs.sim import render_context_bootstrap +from rcs.sim import filament, render_context_bootstrap from rcs import sim +logger = logging.getLogger(__name__) + + +def _camera_id(model: mujoco.MjModel, cfg: SimCameraConfig) -> int: + return mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_CAMERA, cfg.identifier) + + +def _intrinsics( + model: mujoco.MjModel, cfg: SimCameraConfig +) -> np.ndarray[tuple[Literal[3], Literal[4]], np.dtype[np.float64]]: + fovy = model.cam_fovy[_camera_id(model, cfg)] + fx = fy = 0.5 * cfg.resolution_height / np.tan(fovy * np.pi / 360) + return np.array( + [ + [fx, 0, (cfg.resolution_width - 1) / 2, 0], + [0, fy, (cfg.resolution_height - 1) / 2, 0], + [0, 0, 1, 0], + ] + ) + + +def _extrinsics( + model: mujoco.MjModel, data: mujoco.MjData, cfg: SimCameraConfig +) -> np.ndarray[tuple[Literal[4], Literal[4]], np.dtype[np.float64]]: + cam_id = _camera_id(model, cfg) + xpos = data.cam_xpos[cam_id] + xmat = data.cam_xmat[cam_id].reshape(3, 3) + + cam = common.Pose(rotation=xmat, translation=xpos) + # put z axis infront + rotation_p = common.Pose(rpy_vector=np.array([np.pi, 0, 0]), translation=np.array([0, 0, 0])) # type: ignore + cam = cam * rotation_p + + return cam.inverse().pose_matrix() + class SimCameraSet(_SimCameraSet): """Represents a set of cameras in a mujoco simulation. Implements BaseCameraSet + + Rendering happens in C++ with MuJoCo's classic OpenGL renderer. If the simulation was configured with + ``SimConfig(renderer=RendererBackend.FILAMENT)``, constructing this class returns a + :class:`FilamentSimCameraSet` instead. """ + def __new__( + cls, + simulation: sim.Sim, + cameras: dict[str, SimCameraConfig], + physical_units: bool = False, + render_on_demand: bool = True, + ): + if simulation.get_config().renderer == RendererBackend.FILAMENT: + return FilamentSimCameraSet(simulation, cameras, physical_units, render_on_demand) + return super().__new__(cls) + def __init__( self, simulation: sim.Sim, @@ -93,28 +144,10 @@ def _cpp_to_python_frames(self, cpp_frameset: _FrameSet | None) -> FrameSet | No return FrameSet(frames=frames, avg_timestamp=cpp_frameset.timestamp) def _intrinsics(self, camera_name) -> np.ndarray[tuple[Literal[3], Literal[4]], np.dtype[np.float64]]: - cam_id = mujoco.mj_name2id(self._sim.model, mujoco.mjtObj.mjOBJ_CAMERA, self.cameras[camera_name].identifier) - fovy = self._sim.model.cam_fovy[cam_id] - fx = fy = 0.5 * self.cameras[camera_name].resolution_height / np.tan(fovy * np.pi / 360) - return np.array( - [ - [fx, 0, (self.cameras[camera_name].resolution_width - 1) / 2, 0], - [0, fy, (self.cameras[camera_name].resolution_height - 1) / 2, 0], - [0, 0, 1, 0], - ] - ) + return _intrinsics(self._sim.model, self.cameras[camera_name]) def _extrinsics(self, camera_name) -> np.ndarray[tuple[Literal[4], Literal[4]], np.dtype[np.float64]]: - cam_id = mujoco.mj_name2id(self._sim.model, mujoco.mjtObj.mjOBJ_CAMERA, self.cameras[camera_name].identifier) - xpos = self._sim.data.cam_xpos[cam_id] - xmat = self._sim.data.cam_xmat[cam_id].reshape(3, 3) - - cam = common.Pose(rotation=xmat, translation=xpos) - # put z axis infront - rotation_p = common.Pose(rpy_vector=np.array([np.pi, 0, 0]), translation=np.array([0, 0, 0])) # type: ignore - cam = cam * rotation_p - - return cam.inverse().pose_matrix() + return _extrinsics(self._sim.model, self._sim.data, self.cameras[camera_name]) def calibrate(self) -> bool: return True @@ -135,3 +168,169 @@ def camera_names(self) -> list[str]: @property def name_to_identifier(self) -> dict[str, str]: return {name: cfg.identifier for name, cfg in self.cameras.items()} + + +class _ClassicDepthRenderer: + """Renders metric depth with MuJoCo's classic renderer from Python. + + Unlike ``mujoco.Renderer`` this is not limited by the model's ``offwidth``/``offheight``. + """ + + def __init__(self, model: mujoco.MjModel, width: int, height: int): + render_context_bootstrap.require("simulation depth rendering") + render_context_bootstrap.make_current() + self._model = model + self._scene = mujoco.MjvScene(model, maxgeom=2000) + self._opt = mujoco.MjvOption() + self._ctx = mujoco.MjrContext(model, mujoco.mjtFontScale.mjFONTSCALE_150) + mujoco.mjr_setBuffer(mujoco.mjtFramebuffer.mjFB_OFFSCREEN, self._ctx) + mujoco.mjr_resizeOffscreen(width, height, self._ctx) + self._viewport = mujoco.MjrRect(0, 0, width, height) + self._depth = np.empty((height, width), dtype=np.float32) + extent = model.stat.extent + self._near = model.vis.map.znear * extent + self._far = model.vis.map.zfar * extent + + def render(self, data: mujoco.MjData, camera: mujoco.MjvCamera) -> np.ndarray: + """Returns metric depth in meters as (H, W) float32, top row first.""" + render_context_bootstrap.make_current() + mujoco.mjv_updateScene(self._model, data, self._opt, None, camera, mujoco.mjtCatBit.mjCAT_ALL, self._scene) + mujoco.mjr_setBuffer(mujoco.mjtFramebuffer.mjFB_OFFSCREEN, self._ctx) + mujoco.mjr_render(self._viewport, self._scene, self._ctx) + mujoco.mjr_readPixels(None, self._depth, self._viewport, self._ctx) + # OpenGL reads bottom-up; convert the [0, 1] depth buffer to meters (see SimCameraSet) + depth = self._depth[::-1] + return self._near / (1 - depth * (1 - self._near / self._far)) + + def close(self): + self._ctx.free() + + +class FilamentSimCameraSet: + """Set of simulation cameras rendered with MuJoCo's Filament renderer (MuJoCo >= 3.15). + Implements BaseCameraSet + + Color images come from Filament. Filament does not expose a metric depth buffer through + MuJoCo's Python API, so depth images are rendered with the classic renderer; they are always + metric (scaled by ``BaseCameraSet.DEPTH_SCALE``), regardless of ``physical_units``. The depth + pass costs about as much as the classic renderer, set ``render_depth=False`` (done automatically + by ``CameraSetWrapper(include_depth=False)``) to skip it; frames then have ``depth=None``. + + Rendering happens in Python when frames are requested, i.e. always "on demand"; rendering at a + fixed camera frame rate while the simulation steps is not supported with this backend. + """ + + DEPTH_SCALE: int = BaseCameraSet.DEPTH_SCALE + + def __init__( + self, + simulation: sim.Sim, + cameras: dict[str, SimCameraConfig], + physical_units: bool = True, + render_on_demand: bool = True, + max_buffer_frames: int = 100, + render_depth: bool = True, + ): + filament.require("FilamentSimCameraSet") + if not physical_units: + logger.warning("FilamentSimCameraSet always returns metric depth; physical_units=False is ignored.") + if not render_on_demand: + logger.warning("FilamentSimCameraSet always renders on demand; render_on_demand=False is ignored.") + if max_buffer_frames <= 0: + msg = "max_buffer_frames must be positive" + raise ValueError(msg) + self._sim = simulation + self.cameras = cameras + self.physical_units = physical_units + self.render_on_demand = True + self.render_depth = render_depth + self._buffer: list[FrameSet] = [] + self._max_buffer_frames = max_buffer_frames + + self._renderer = filament.FilamentRenderer(self._sim.model) + self._mj_cameras: dict[str, mujoco.MjvCamera] = {} + for name, cfg in cameras.items(): + cam = mujoco.MjvCamera() + if cfg.type == CameraType.default_free: + mujoco.mjv_defaultFreeCamera(self._sim.model, cam) + else: + cam.type = int(cfg.type) + cam.fixedcamid = _camera_id(self._sim.model, cfg) + self._mj_cameras[name] = cam + # classic renderers for metric depth, one per resolution + self._depth_renderers: dict[tuple[int, int], _ClassicDepthRenderer] = {} + + def _depth_renderer(self, cfg: SimCameraConfig) -> _ClassicDepthRenderer: + key = (cfg.resolution_width, cfg.resolution_height) + if key not in self._depth_renderers: + self._depth_renderers[key] = _ClassicDepthRenderer(self._sim.model, *key) + return self._depth_renderers[key] + + def _render(self) -> FrameSet: + model, data = self._sim.model, self._sim.data + timestamp = data.time + self._renderer.update(data) + frames: dict[str, Frame] = {} + for name, cfg in self.cameras.items(): + color = self._renderer.render(data, self._mj_cameras[name], cfg.resolution_width, cfg.resolution_height) + intrinsics = _intrinsics(model, cfg) + extrinsics = _extrinsics(model, data, cfg) + depth_frame = None + if self.render_depth: + depth = self._depth_renderer(cfg).render(data, self._mj_cameras[name])[..., np.newaxis] + depth_frame = DataFrame( + data=(depth * BaseCameraSet.DEPTH_SCALE).astype(np.uint16), + timestamp=timestamp, + intrinsics=intrinsics, + extrinsics=extrinsics, + ) + frames[name] = Frame( + camera=CameraFrame( + color=DataFrame(data=color, timestamp=timestamp, intrinsics=intrinsics, extrinsics=extrinsics), + depth=depth_frame, + ), + avg_timestamp=timestamp, + ) + return FrameSet(frames=frames, avg_timestamp=timestamp) + + def buffer_size(self) -> int: + return len(self._buffer) + + def clear_buffer(self): + self._buffer.clear() + + def get_latest_frames(self) -> FrameSet | None: + """Renders all cameras for the current simulation state and returns the frames.""" + if self._buffer and self._buffer[-1].avg_timestamp == self._sim.data.time: + return self._buffer[-1] + frameset = self._render() + self._buffer.append(frameset) + del self._buffer[: -self._max_buffer_frames] + return frameset + + def get_timestamp_frames(self, ts: datetime) -> FrameSet | None: + """Returns the most recent buffered frames with a simulation time <= ts.""" + for frameset in reversed(self._buffer): + if frameset.avg_timestamp is not None and frameset.avg_timestamp <= ts.timestamp(): + return frameset + return None + + def calibrate(self) -> bool: + return True + + def config(self, camera_name: str) -> SimCameraConfig: + return self.cameras[camera_name] + + def close(self): + for renderer in self._depth_renderers.values(): + renderer.close() + self._depth_renderers.clear() + self._renderer.close() + + @property + def camera_names(self) -> list[str]: + return list(self.cameras.keys()) + + @property + def name_to_identifier(self) -> dict[str, str]: + return {name: cfg.identifier for name, cfg in self.cameras.items()} diff --git a/python/rcs/envs/base.py b/python/rcs/envs/base.py index 0d1b6261..f82a812b 100644 --- a/python/rcs/envs/base.py +++ b/python/rcs/envs/base.py @@ -961,6 +961,9 @@ def __init__(self, env, camera_set: BaseCameraSet, include_depth: bool = False): super().__init__(env) self.camera_set = camera_set self.include_depth = include_depth + # camera sets that render depth separately (e.g. FilamentSimCameraSet) can skip it if not requested + if hasattr(camera_set, "render_depth"): + camera_set.render_depth = include_depth self.observation_space: gym.spaces.Dict # rgb is always included diff --git a/python/rcs/sim/__init__.py b/python/rcs/sim/__init__.py index dab8c835..58a7b550 100644 --- a/python/rcs/sim/__init__.py +++ b/python/rcs/sim/__init__.py @@ -1,4 +1,5 @@ from rcs._core.sim import ( + RendererBackend, SimCameraConfig, SimConfig, SimGripper, @@ -27,4 +28,5 @@ "gui_loop", "SimCameraConfig", "SimConfig", + "RendererBackend", ] diff --git a/python/rcs/sim/filament.py b/python/rcs/sim/filament.py new file mode 100644 index 00000000..cbdd1c32 --- /dev/null +++ b/python/rcs/sim/filament.py @@ -0,0 +1,235 @@ +""" +Filament rendering backend (MuJoCo >= 3.15). + +MuJoCo ships its Filament-based renderer only inside the Python extension modules +(``mujoco._render_filament`` and ``mujoco.experimental.studio``); ``libmujoco`` does not +export the ``mjrf_*`` C API. Everything Filament related therefore lives on the Python +side of RCS: + +- :class:`FilamentRenderer` renders RGB images of fixed/free MuJoCo cameras offscreen. + Filament does not expose a metric depth buffer through the Python API (its depth draw + mode is an 8-bit visualization), so depth is still produced by the classic renderer. +- :func:`gui_loop` runs MuJoCo Studio (the Filament viewer) in the RCS GUI subprocess. + Studio must own the main thread of its process and, unlike ``mujoco.viewer``, must *not* + run under ``mjpython`` on macOS. + +Enable it via ``SimConfig(renderer=RendererBackend.FILAMENT)``. + +Environment variables: + +- ``RCS_FILAMENT_GRAPHICS_API``: ``opengl`` or ``vulkan`` (default: opengl) +- ``RCS_FILAMENT_SOFTWARE_RENDERING=1``: force software rendering for offscreen cameras +- ``RCS_FILAMENT_GFX``: Studio graphics mode for the GUI, e.g. ``opengl``, ``vulkan``, ``web`` + (``web`` serves the viewer over http instead of opening a native window) +""" + +import os +import sys +import threading +from logging import getLogger +from tempfile import NamedTemporaryFile + +import mujoco +import numpy as np +from rcs._core.sim import GuiClient as _GuiClient +from rcs.utils import SimpleFrameRate + +logger = getLogger(__name__) + +# Target frames per second of the GUI loop +FPS = 60 + + +def _import_error() -> str | None: + # NOTE: do not import mujoco.experimental.studio.window here: it registers Filament resource + # providers that MuJoCo Studio registers again, which crashes the GUI process (see gui_loop) + try: + import mujoco._render_filament + import mujoco.experimental.studio # noqa: F401 + except ImportError as exc: + return repr(exc) + return None + + +def is_available() -> bool: + return _import_error() is None + + +def require(feature: str = "Filament rendering"): + error = _import_error() + if error is None: + return + msg = ( + f"{feature} requires the Filament renderer, which is shipped with the MuJoCo Python package " + f"since version 3.15 (installed: {mujoco.__version__}). Import failed with: {error}" + ) + raise RuntimeError(msg) + + +class FilamentRenderer: + """Offscreen RGB renderer for a MuJoCo model based on Filament. + + All methods must be called from the thread that created the renderer: Filament asserts + thread affinity when its engine is destroyed. + """ + + def __init__(self, model: mujoco.MjModel): + require("FilamentRenderer") + import mujoco._render_filament as mjrf # type: ignore[import-not-found] + + # importing the studio window module registers Filament's built-in assets + # (materials such as pbr.filamat); without it context creation fails. It must not be + # imported in a process that runs MuJoCo Studio (see _import_error) + import mujoco.experimental.studio.window # noqa: F401 + + self._mjrf = mjrf + self.model = model + self._ctx = mjrf.Context(self._context_config()) + self._ctx.set_clear_color(np.zeros(3, dtype=np.float32)) + self._objects = mjrf.ModelObjects(self._ctx, model) + self._scene = self._ctx.create_scene(mjrf.SceneParams()) + self._scene.configure_from_model(model) + self._lights = mjrf.ModelLights(self._scene, self._objects) + self._renderables = mjrf.ModelRenderables(self._scene, self._objects) + # one render target per resolution, created lazily + self._targets: dict[tuple[int, int], tuple[object, object, np.ndarray]] = {} + self._closed = False + + def _context_config(self): + mjrf = self._mjrf + cfg = mjrf.ContextConfig() + # OpenGL like MuJoCo's own samples; the platform default resolves to Vulkan on some systems + # (e.g. macOS) where it is not usable + cfg.graphics_api = mjrf.GraphicsApi.GRAPHICS_API_OPENGL + api = os.environ.get("RCS_FILAMENT_GRAPHICS_API", "").lower() + if api == "vulkan": + cfg.graphics_api = mjrf.GraphicsApi.GRAPHICS_API_VULKAN + elif api not in ("", "opengl"): + logger.warning("Unknown RCS_FILAMENT_GRAPHICS_API=%r, using OpenGL", api) + cfg.force_software_rendering = os.environ.get("RCS_FILAMENT_SOFTWARE_RENDERING", "") == "1" + return cfg + + def set_options(self, opt: mujoco.MjvOption): + self._renderables.set_options(opt) + + def update(self, data: mujoco.MjData): + """Syncs lights and renderables with the current simulation state.""" + self._lights.update(data) + self._renderables.update(data) + + def _target(self, width: int, height: int): + key = (width, height) + if key not in self._targets: + mjrf = self._mjrf + target = self._ctx.create_render_target( + mjrf.RenderTargetConfig(color_format=mjrf.PixelFormat.PIXEL_FORMAT_RGB8) + ) + target.resize(width, height) + buffer = np.empty((height, width, 3), dtype=np.uint8) + read = mjrf.ReadPixelsRequest() + read.target = target + read.set_buffer(buffer) + self._targets[key] = (target, read, buffer) + return self._targets[key] + + def render(self, data: mujoco.MjData, camera: mujoco.MjvCamera, width: int, height: int) -> np.ndarray: + """Renders the given camera and returns an (H, W, 3) uint8 RGB image (top row first). + + Call :meth:`update` first whenever ``data`` changed. + """ + mjrf = self._mjrf + target, read, buffer = self._target(width, height) + glcam = mujoco.mjv_camera2GLCamera(self.model, data, camera) + request = mjrf.RenderRequest( + camera=mjrf.Camera( + pos=glcam.pos, + forward=glcam.forward, + up=glcam.up, + frustum_bottom=glcam.frustum_bottom, + frustum_top=glcam.frustum_top, + frustum_near=glcam.frustum_near, + frustum_far=glcam.frustum_far, + ), + draw_mode=mjrf.DrawMode.DRAW_MODE_DEFAULT, + ) + request.scene = self._scene + request.target = target + request.viewport.left = 0 + request.viewport.bottom = 0 + request.viewport.width = width + request.viewport.height = height + # the bindings currently support only a single read request per render call + frame = self._ctx.render([request], [read]) + self._ctx.wait_for_frame(frame) + return buffer.copy() + + def close(self): + if self._closed: + return + self._closed = True + # destroy in reverse creation order; the context must outlive everything it created + self._targets.clear() + del self._renderables, self._lights, self._scene, self._objects + del self._ctx + + +def gui_executable() -> str: + """Python executable for the Filament GUI subprocess (never mjpython, see module docstring).""" + return sys.executable + + +def gui_loop(gui_uuid: str, close_event): + """Runs MuJoCo Studio for the simulation identified by ``gui_uuid``. + + Mirrors :func:`rcs.sim.sim.gui_loop`: the GUI process holds its own copy of the model and data + and synchronizes the state through the GuiClient. The viewer runs on the main thread of this + process, stepping/syncing happens on a background thread. + """ + require("The Filament GUI") + from mujoco.experimental.studio import ( + launch_native, + launch_thread, + viewer_app, + viewer_handle, + viewer_protocol, + ) + + gui_client = _GuiClient(gui_uuid) + model_bytes = gui_client.get_model_bytes() + with NamedTemporaryFile(mode="wb") as f: + f.write(model_bytes) + f.flush() + model = mujoco.MjModel.from_binary_path(f.name) + data = mujoco.MjData(model) + gui_client.set_model_and_data(model._address, data._address) + mujoco.mj_step(model, data) + + viewer_endpoint, sim_endpoint = launch_thread.make_thread_endpoints() + handle = viewer_handle.ViewerHandle(sim_endpoint) + viewer_closed = threading.Event() + + def sim_loop(): + frame_rate = SimpleFrameRate(FPS, "gui_loop") + try: + while not close_event.is_set() and not viewer_closed.is_set() and handle.is_running(): + mujoco.mj_step(model, data) + # the viewer may hand back a new model (e.g. drag and drop); we keep ours as the + # GuiClient owns the pointers to model and data + handle.sync(model, data) + gui_client.sync() + frame_rate() + finally: + handle.close() + + sim_thread = threading.Thread(target=sim_loop, daemon=True) + sim_thread.start() + try: + config = viewer_protocol.ViewerConfig(title="RCS", gfx=os.environ.get("RCS_FILAMENT_GFX", "")) + # ViewerApp provides the interactive Studio GUI (camera selection, visualization options, + # dragging bodies, ...); without it only the bare scene is rendered. The camera selection + # lives in the toolbar, which Studio hides by default ([ and ] cycle cameras regardless). + app = viewer_app.ViewerApp(viewer_app.ViewerAppConfig(show_toolbar=True)) + launch_native.run_native_viewer(config, viewer_endpoint, plugins=[app]) + finally: + viewer_closed.set() + sim_thread.join(timeout=5) diff --git a/python/rcs/sim/render_context_bootstrap.py b/python/rcs/sim/render_context_bootstrap.py index e27f2411..ff1bb065 100644 --- a/python/rcs/sim/render_context_bootstrap.py +++ b/python/rcs/sim/render_context_bootstrap.py @@ -95,6 +95,13 @@ def require(feature: str = "offscreen rendering"): raise RuntimeError(message) +def make_current(): + """Makes the bootstrapped GL context current on the calling thread (for rendering from Python).""" + require() + assert _state.gl_context is not None + _state.gl_context.make_current() + + def bootstrap(): if not _state.available: return diff --git a/python/rcs/sim/sim.py b/python/rcs/sim/sim.py index 4ee8b02b..9bccf712 100644 --- a/python/rcs/sim/sim.py +++ b/python/rcs/sim/sim.py @@ -19,7 +19,7 @@ from rcs._core.sim import DynamicJointSchema, DynamicJointState from rcs._core.sim import GuiClient as _GuiClient from rcs._core.sim import Sim as _Sim -from rcs.sim import SimConfig, render_context_bootstrap +from rcs.sim import RendererBackend, SimConfig, filament, render_context_bootstrap from rcs.sim.composer import ModelComposer from rcs.utils import SimpleFrameRate @@ -33,9 +33,19 @@ ROOT_RELATIVE_FREE_STATE_ENCODING = "root_relative_free" -def configure_viewer_mp_context(ctx: "mp.context.SpawnContext") -> None: - """On macOS the passive MuJoCo viewer must run under ``mjpython``.""" +def configure_viewer_mp_context( + ctx: "mp.context.SpawnContext", renderer: RendererBackend = RendererBackend.CLASSIC +) -> None: + """Selects the interpreter of the GUI subprocess. + + On macOS the passive MuJoCo viewer must run under ``mjpython``, whereas MuJoCo Studio (Filament) must + run under a regular ``python`` as it owns the main thread itself. + """ + if renderer == RendererBackend.FILAMENT: + ctx.set_executable(filament.gui_executable()) + return if sys.platform != "darwin": + ctx.set_executable(sys.executable) return mjpython = shutil.which("mjpython") if mjpython is None: @@ -79,7 +89,6 @@ def __init__(self, mjmdl: str | PathLike | ModelComposer, cfg: SimConfig | None self.data = mj.MjData(self.model) super().__init__(self.model._address, self.data._address) self._mp_context = mp.get_context("spawn") - configure_viewer_mp_context(self._mp_context) self._gui_uuid: Optional[str] = None self._gui_client: Optional[_GuiClient] = None self._gui_process: Optional[mp.context.SpawnProcess] = None @@ -250,13 +259,22 @@ def close_gui(self): self._gui_atexit_registered = False def open_gui(self): + """Opens the viewer in a subprocess. + + Uses the passive ``mujoco.viewer`` or, with ``SimConfig(renderer=RendererBackend.FILAMENT)``, + MuJoCo Studio (Filament). + """ + renderer = self.get_config().renderer + if renderer == RendererBackend.FILAMENT: + filament.require("The Filament GUI") if self._gui_uuid is None: self._gui_uuid = "rcs_" + str(uuid.uuid4()) self._start_gui_server(self._gui_uuid) if self._gui_process is None or not self._gui_process.is_alive(): + configure_viewer_mp_context(self._mp_context, renderer) self._stop_event = self._mp_context.Event() self._gui_process = self._mp_context.Process( - target=gui_loop, + target=filament.gui_loop if renderer == RendererBackend.FILAMENT else gui_loop, args=(self._gui_uuid, self._stop_event), daemon=True, ) diff --git a/python/tests/test_filament.py b/python/tests/test_filament.py new file mode 100644 index 00000000..369646ae --- /dev/null +++ b/python/tests/test_filament.py @@ -0,0 +1,92 @@ +import gymnasium as gym +import numpy as np +import pytest +from rcs.camera.sim import FilamentSimCameraSet, SimCameraSet +from rcs.envs.base import CameraSetWrapper +from rcs.envs.configs import EmptyWorldFR3 +from rcs.sim import RendererBackend, SimConfig, filament + +from rcs import sim + +pytestmark = pytest.mark.skipif(not filament.is_available(), reason="Filament renderer requires mujoco >= 3.15") + + +@pytest.fixture() +def fr3_sim(): + scene = EmptyWorldFR3() + cfg = scene.prefixed_cfg(scene.config()) + simulation = sim.Sim(scene.create_model(cfg), SimConfig(renderer=RendererBackend.FILAMENT)) + return simulation, cfg + + +def test_sim_camera_set_dispatches_to_filament(fr3_sim): + simulation, cfg = fr3_sim + camera_set = SimCameraSet(simulation, cfg.camera_cfgs, physical_units=True) + assert isinstance(camera_set, FilamentSimCameraSet) + assert camera_set.camera_names == list(cfg.camera_cfgs.keys()) + camera_set.close() + + +def test_filament_camera_set_renders_rgb_and_metric_depth(fr3_sim): + simulation, cfg = fr3_sim + camera_set = FilamentSimCameraSet(simulation, cfg.camera_cfgs) + try: + simulation.step(1) + frameset = camera_set.get_latest_frames() + assert frameset is not None + assert frameset.frames.keys() == cfg.camera_cfgs.keys() + for name, frame in frameset.frames.items(): + cam_cfg = cfg.camera_cfgs[name] + assert frame.camera.depth is not None + color = frame.camera.color.data + depth = frame.camera.depth.data + assert color.shape == (cam_cfg.resolution_height, cam_cfg.resolution_width, 3) + assert color.dtype == np.uint8 + # the scene is lit and textured, so the image must not be uniform + assert color.std() > 1 + assert depth.shape == (cam_cfg.resolution_height, cam_cfg.resolution_width, 1) + assert depth.dtype == np.uint16 + # metric depth in millimeters: the cameras look at the robot/floor within a few meters + assert 10 < np.median(depth) < 10_000 + assert frame.camera.color.intrinsics is not None + assert frame.camera.color.extrinsics is not None + assert frame.camera.color.intrinsics.shape == (3, 4) + assert frame.camera.color.extrinsics.shape == (4, 4) + + # the same simulation time returns the buffered frames, a new step renders again + assert camera_set.get_latest_frames() is frameset + simulation.step(1) + assert camera_set.get_latest_frames() is not frameset + assert camera_set.buffer_size() == 2 + finally: + camera_set.close() + + +def test_filament_camera_set_skips_depth_when_not_requested(fr3_sim): + simulation, cfg = fr3_sim + camera_set = FilamentSimCameraSet(simulation, cfg.camera_cfgs, render_depth=False) + try: + simulation.step(1) + frameset = camera_set.get_latest_frames() + assert frameset is not None + assert all(frame.camera.depth is None for frame in frameset.frames.values()) + assert all(frame.camera.color.data.dtype == np.uint8 for frame in frameset.frames.values()) + finally: + camera_set.close() + + +class _EmptyObsEnv(gym.Env): + observation_space = gym.spaces.Dict({}) + action_space = gym.spaces.Dict({}) + + +def test_camera_set_wrapper_configures_depth_rendering(fr3_sim): + simulation, cfg = fr3_sim + camera_set = FilamentSimCameraSet(simulation, cfg.camera_cfgs) + try: + CameraSetWrapper(_EmptyObsEnv(), camera_set, include_depth=False) + assert camera_set.render_depth is False + CameraSetWrapper(_EmptyObsEnv(), camera_set, include_depth=True) + assert camera_set.render_depth is True + finally: + camera_set.close() diff --git a/src/pybind/rcs.cpp b/src/pybind/rcs.cpp index 307137ff..ee731bfe 100644 --- a/src/pybind/rcs.cpp +++ b/src/pybind/rcs.cpp @@ -720,27 +720,36 @@ PYBIND11_MODULE(_core, m) { .def_readonly("last_width", &rcs::sim::SimGripperState::last_width) .def_readonly("collision", &rcs::sim::SimGripperState::collision); + py::enum_(sim, "RendererBackend") + .value("CLASSIC", rcs::sim::RendererBackend::CLASSIC) + .value("FILAMENT", rcs::sim::RendererBackend::FILAMENT) + .export_values(); + rcs::sim::SimConfig default_sim_cfg = rcs::sim::SimConfig(); py::class_(sim, "SimConfig") .def(py::init([](bool async_control, bool realtime, double frequency, - int max_convergence_steps) { + int max_convergence_steps, + rcs::sim::RendererBackend renderer) { rcs::sim::SimConfig config; config.async_control = async_control; config.realtime = realtime; config.frequency = frequency; config.max_convergence_steps = max_convergence_steps; + config.renderer = renderer; return config; }), py::arg("async_control") = default_sim_cfg.async_control, py::arg("realtime") = default_sim_cfg.realtime, py::arg("frequency") = default_sim_cfg.frequency, py::arg("max_convergence_steps") = - default_sim_cfg.max_convergence_steps) + default_sim_cfg.max_convergence_steps, + py::arg("renderer") = default_sim_cfg.renderer) .def_readwrite("async_control", &rcs::sim::SimConfig::async_control) .def_readwrite("realtime", &rcs::sim::SimConfig::realtime) .def_readwrite("frequency", &rcs::sim::SimConfig::frequency) .def_readwrite("max_convergence_steps", &rcs::sim::SimConfig::max_convergence_steps) + .def_readwrite("renderer", &rcs::sim::SimConfig::renderer) .def("__copy__", [](const rcs::sim::SimConfig& self) { return rcs::sim::SimConfig(self); diff --git a/src/sim/sim.h b/src/sim/sim.h index 6cae264c..fa8d8a87 100644 --- a/src/sim/sim.h +++ b/src/sim/sim.h @@ -29,11 +29,15 @@ class Renderer { std::unordered_map ctxs; }; +// Filament renderer is implemented in python +enum class RendererBackend { CLASSIC, FILAMENT }; + struct SimConfig { bool async_control = false; bool realtime = false; double frequency = 30; // in Hz int max_convergence_steps = 500; + RendererBackend renderer = RendererBackend::CLASSIC; }; struct Callback {