Adds an example showcasing different camera types (#565)

# Description

This MR fixes some of the docstrings related to the tile-rendering
camera. Additionally, it adds a demo script showcasing the images
obtained through the different camera implementations in IsaacLab.

## Type of change

- New feature (non-breaking change which adds functionality)
- This change requires a documentation update

## Checklist

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] I have run all the tests with `./isaaclab.sh --test` and they pass
- [ ] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

Author: Mayankm96

docs/source/api/lab/omni.isaac.lab.sensors.rst

@@ -18,6 +18,8 @@
     Camera
     CameraData
     CameraCfg
+    TiledCamera
+    TiledCameraCfg
     ContactSensor
     ContactSensorData
     ContactSensorCfg
@@ -29,8 +31,6 @@
     RayCasterCfg
     RayCasterCamera
     RayCasterCameraCfg
-    TiledCamera
-    TiledCameraCfg
 
 Sensor Base
 -----------
@@ -61,6 +61,20 @@ USD Camera
     :show-inheritance:
     :exclude-members: __init__, class_type
 
+Tile-Rendered USD Camera
+------------------------
+
+.. autoclass:: TiledCamera
+    :members:
+    :inherited-members:
+    :show-inheritance:
+
+.. autoclass:: TiledCameraCfg
+    :members:
+    :inherited-members:
+    :show-inheritance:
+    :exclude-members: __init__, class_type
+
 Contact Sensor
 --------------
 
@@ -80,7 +94,6 @@ Contact Sensor
     :show-inheritance:
     :exclude-members: __init__, class_type
 
-
 Frame Transformer
 -----------------
 
@@ -137,17 +150,3 @@ Ray-Cast Camera
     :inherited-members:
     :show-inheritance:
     :exclude-members: __init__, class_type
-
-Tiled Rendering
----------------
-
-.. autoclass:: TiledCamera
-    :members:
-    :inherited-members:
-    :show-inheritance:
-
-.. autoclass:: TiledCameraCfg
-    :members:
-    :inherited-members:
-    :show-inheritance:
-    :exclude-members: __init__, class_type

source/extensions/omni.isaac.lab/config/extension.toml

@@ -1,7 +1,7 @@
 [package]
 
 # Note: Semantic Versioning is used: https://semver.org/
-version = "0.18.1"
+version = "0.18.2"
 
 # Description
 title = "Isaac Lab framework for Robot Learning"

source/extensions/omni.isaac.lab/docs/CHANGELOG.rst

@@ -1,15 +1,25 @@
 Changelog
 ---------
 
+0.18.2 (2024-06-25)
+~~~~~~~~~~~~~~~~~~~
+
+Changed
+^^^^^^^
+
+* Moved the configuration for tile-rendered camera into its own file named ``tiled_camera_cfg.py``.
+  This makes it easier to follow where the configuration is located and how it is related to the class.
+
+
 0.18.1 (2024-06-25)
 ~~~~~~~~~~~~~~~~~~~
 
 Changed
 ^^^^^^^
 
-* Ensured that a parity between class and its configuration class is explicitly visible in the :class:`omni.isaac.lab.envs`
-  module. This makes it easier to follow where definitions are located and how they are related. This should not be
-  a breaking change as the classes are still accessible through the same module.
+* Ensured that a parity between class and its configuration class is explicitly visible in the
+  :mod:`omni.isaac.lab.envs` module. This makes it easier to follow where definitions are located and how
+  they are related. This should not be a breaking change as the classes are still accessible through the same module.
 
 
 0.18.0 (2024-06-13)

source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/__init__.py

@@ -6,7 +6,8 @@
 """Sub-module for camera wrapper around USD camera prim."""
 
 from .camera import Camera
-from .camera_cfg import CameraCfg, TiledCameraCfg
+from .camera_cfg import CameraCfg
 from .camera_data import CameraData
 from .tiled_camera import TiledCamera
+from .tiled_camera_cfg import TiledCameraCfg
 from .utils import *  # noqa: F401, F403

source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/camera_cfg.py

@@ -11,7 +11,6 @@
 
 from ..sensor_base_cfg import SensorBaseCfg
 from .camera import Camera
-from .tiled_camera import TiledCamera
 
 
 @configclass
@@ -107,64 +106,3 @@ class OffsetCfg:
     If True, instance segmentation is converted to an image where instance IDs are mapped to colors.
     and returned as a ``uint8`` 4-channel array. If False, the output is returned as a ``int32`` array.
     """
-
-
-@configclass
-class TiledCameraCfg(SensorBaseCfg):
-    """Configuration for a tiled rendering camera sensor."""
-
-    @configclass
-    class OffsetCfg:
-        """The offset pose of the sensor's frame from the sensor's parent frame."""
-
-        pos: tuple[float, float, float] = (0.0, 0.0, 0.0)
-        """Translation w.r.t. the parent frame. Defaults to (0.0, 0.0, 0.0)."""
-
-        rot: tuple[float, float, float, float] = (1.0, 0.0, 0.0, 0.0)
-        """Quaternion rotation (w, x, y, z) w.r.t. the parent frame. Defaults to (1.0, 0.0, 0.0, 0.0)."""
-
-        convention: Literal["opengl", "ros", "world"] = "ros"
-        """The convention in which the frame offset is applied. Defaults to "ros".
-
-        - ``"opengl"`` - forward axis: ``-Z`` - up axis: ``+Y`` - Offset is applied in the OpenGL (Usd.Camera) convention.
-        - ``"ros"``    - forward axis: ``+Z`` - up axis: ``-Y`` - Offset is applied in the ROS convention.
-        - ``"world"``  - forward axis: ``+X`` - up axis: ``+Z`` - Offset is applied in the World Frame convention.
-
-        """
-
-    class_type: type = TiledCamera
-
-    offset: OffsetCfg = OffsetCfg()
-    """The offset pose of the sensor's frame from the sensor's parent frame. Defaults to identity.
-
-    Note:
-        The parent frame is the frame the sensor attaches to. For example, the parent frame of a
-        camera at path ``/World/envs/env_0/Robot/Camera`` is ``/World/envs/env_0/Robot``.
-    """
-
-    spawn: PinholeCameraCfg | FisheyeCameraCfg | None = MISSING
-    """Spawn configuration for the asset.
-
-    If None, then the prim is not spawned by the asset. Instead, it is assumed that the
-    asset is already present in the scene.
-    """
-
-    data_types: list[str] = ["rgb"]
-    """List of sensor names/types to enable for the camera. Defaults to ["rgb"].
-
-    Please refer to the :class:`Camera` class for a list of available data types.
-    """
-
-    width: int = MISSING
-    """Width of the image in pixels."""
-
-    height: int = MISSING
-    """Height of the image in pixels."""
-
-    return_latest_camera_pose: bool = False
-    """Whether to return the latest camera pose when fetching the camera's data. Defaults to False.
-
-    If True, the latest camera pose is returned in the camera's data which will slow down performance
-    due to the use of :class:`XformPrimView`.
-    If False, the pose of the camera during initialization is returned.
-    """

source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py

@@ -27,17 +27,27 @@
 
 
 class TiledCamera(Camera):
-    r"""The tiled rendering camera sensor for acquiring RGB and depth data.
+    r"""The tiled rendering based camera sensor for acquiring RGB and depth data.
 
-    This class wraps over the `UsdGeom Camera`_ for providing a consistent API for acquiring visual data.
-    It ensures that the camera follows the ROS convention for the coordinate system.
+    This class inherits from the :class:`Camera` class but uses the tiled-rendering API from Replicator to acquire
+    the visual data. Tiled-rendering concatenates the rendered images from multiple cameras into a single image.
+    This allows for rendering multiple cameras in parallel and is useful for rendering large scenes with multiple
+    cameras efficiently.
 
     The following sensor types are supported:
 
     - ``"rgb"``: A rendered color image.
     - ``"depth"``: An image containing the distance to camera optical center.
 
-    .. _USDGeom Camera: https://graphics.pixar.com/usd/docs/api/class_usd_geom_camera.html
+    .. versionadded:: Isaac Sim 4.0
+
+        This feature is available starting from Isaac Sim 4.0. Before this version, the tiled rendering APIs
+        were not available.
+
+    .. attention::
+        Please note that the fidelity of RGB images may be lower than the standard camera sensor due to the
+        tiled rendering process. Various ray tracing effects such as reflections, refractions, and shadows may not be
+        accurately captured in the RGB images. We are currently working on improving the fidelity of the RGB images.
 
     """
 
@@ -61,7 +71,9 @@ def __init__(self, cfg: TiledCameraCfg):
 
     def __del__(self):
         """Unsubscribes from callbacks and detach from the replicator registry."""
+        # unsubscribe from callbacks
         SensorBase.__del__(self)
+        # detach from the replicator registry
         self._annotator.detach(self.render_product_paths)
 
     def __str__(self) -> str:
@@ -86,9 +98,10 @@ def reset(self, env_ids: Sequence[int] | None = None):
             )
         # reset the timestamps
         SensorBase.reset(self, env_ids)
+        # resolve None
         if env_ids is None:
-            env_ids = self._ALL_INDICES
-        # Reset the frame count
+            env_ids = slice(None)
+        # reset the frame count
         self._frame[env_ids] = 0
 
     """
@@ -142,37 +155,38 @@ def _initialize_impl(self):
             sensor_prim = UsdGeom.Camera(cam_prim)
             self._sensor_prims.append(sensor_prim)
 
+        # start the orchestrator (if not already started)
         rep.orchestrator._orchestrator._is_started = True
-        sensor = rep.create.tiled_sensor(
+        # Create a tiled sensor from the camera prims
+        rep_sensor = rep.create.tiled_sensor(
             cameras=self._view.prim_paths,
             camera_resolution=[self.image_shape[1], self.image_shape[0]],
             tiled_resolution=self._tiled_image_shape(),
             output_types=self.cfg.data_types,
         )
-        render_prod_path = rep.create.render_product(camera=sensor, resolution=self._tiled_image_shape())
+        # Get render product
+        render_prod_path = rep.create.render_product(camera=rep_sensor, resolution=self._tiled_image_shape())
         if not isinstance(render_prod_path, str):
             render_prod_path = render_prod_path.path
         self._render_product_paths = [render_prod_path]
+        # Attach the annotator
         self._annotator = rep.AnnotatorRegistry.get_annotator("RtxSensorGpu", device=self.device, do_array_copy=False)
         self._annotator.attach(self._render_product_paths)
+
         # Create internal buffers
         self._create_buffers()
 
-    def _create_annotator_data(self):
-        raise RuntimeError("Annotator data is not available for the tiled camera sensor.")
-
-    def _process_annotator_output(self, name: str, output: Any) -> tuple[torch.tensor, dict | None]:
-        raise RuntimeError("Annotator data is not available for the tiled camera sensor.")
-
     def _update_buffers_impl(self, env_ids: Sequence[int]):
         # Increment frame count
         self._frame[env_ids] += 1
+
         # Extract the flattened image buffer
         tiled_data_buffer = self._annotator.get_data()
         if isinstance(tiled_data_buffer, np.ndarray):
             tiled_data_buffer = wp.array(tiled_data_buffer, device=self.device)
         else:
             tiled_data_buffer = tiled_data_buffer.to(device=self.device)
+
         # The offset is needed when the buffer contains rgb and depth (the buffer has RGB data first and then depth)
         offset = self._data.output["rgb"].numel() if "rgb" in self.cfg.data_types else 0
         for data_type in self.cfg.data_types:
@@ -193,17 +207,6 @@ def _update_buffers_impl(self, env_ids: Sequence[int]):
     Private Helpers
     """
 
-    def _tiled_image_shape(self) -> tuple[int, int]:
-        """A tuple containing the dimension of the tiled image."""
-        cols, rows = self._tiling_grid_shape()
-        return (self.cfg.width * cols, self.cfg.height * rows)
-
-    def _tiling_grid_shape(self) -> tuple[int, int]:
-        """A tuple containing the tiling grid dimension."""
-        cols = round(math.sqrt(self._view.count))
-        rows = math.ceil(self._view.count / cols)
-        return (cols, rows)
-
     def _check_supported_data_types(self, cfg: TiledCameraCfg):
         """Checks if the data types are supported by the camera."""
         if not set(cfg.data_types).issubset(TiledCamera.SUPPORTED_TYPES):
@@ -235,6 +238,25 @@ def _create_buffers(self):
             ).contiguous()
         self._data.output = TensorDict(data_dict, batch_size=self._view.count, device=self.device)
 
+    def _tiled_image_shape(self) -> tuple[int, int]:
+        """Returns a tuple containing the dimension of the tiled image."""
+        cols, rows = self._tiling_grid_shape()
+        return (self.cfg.width * cols, self.cfg.height * rows)
+
+    def _tiling_grid_shape(self) -> tuple[int, int]:
+        """Returns a tuple containing the tiling grid dimension."""
+        cols = round(math.sqrt(self._view.count))
+        rows = math.ceil(self._view.count / cols)
+        return (cols, rows)
+
+    def _create_annotator_data(self):
+        # we do not need to create annotator data for the tiled camera sensor
+        raise RuntimeError("This function should not be called for the tiled camera sensor.")
+
+    def _process_annotator_output(self, name: str, output: Any) -> tuple[torch.tensor, dict | None]:
+        # we do not need to process annotator output for the tiled camera sensor
+        raise RuntimeError("This function should not be called for the tiled camera sensor.")
+
     """
     Internal simulation callbacks.
     """