[Enhance] Support S3DIS full ceph training (#1542)

* add s3dis ceph support

* modify config

* fix config

configs/_base_/datasets/s3dis_seg-3d-13class.py

@@ -3,12 +3,25 @@ dataset_type = 'S3DISSegDataset'
 data_root = './data/s3dis/'
 class_names = ('ceiling', 'floor', 'wall', 'beam', 'column', 'window', 'door',
                'table', 'chair', 'sofa', 'bookcase', 'board', 'clutter')
+file_client_args = dict(backend='disk')
+# Uncomment the following if use ceph or other file clients.
+# See https://mmcv.readthedocs.io/en/latest/api.html#mmcv.fileio.FileClient
+# for more details.
+# file_client_args = dict(
+#     backend='petrel',
+#     path_mapping=dict({
+#         './data/s3dis/':
+#         's3://openmmlab/datasets/detection3d/s3dis_processed/',
+#         'data/s3dis/':
+#         's3://openmmlab/datasets/detection3d/s3dis_processed/'
+#     }))
 num_points = 4096
 train_area = [1, 2, 3, 4, 6]
 test_area = 5
 train_pipeline = [
     dict(
         type='LoadPointsFromFile',
+        file_client_args=file_client_args,
         coord_type='DEPTH',
         shift_height=False,
         use_color=True,
@@ -16,6 +29,7 @@ train_pipeline = [
         use_dim=[0, 1, 2, 3, 4, 5]),
     dict(
         type='LoadAnnotations3D',
+        file_client_args=file_client_args,
         with_bbox_3d=False,
         with_label_3d=False,
         with_mask_3d=False,
@@ -39,6 +53,7 @@ train_pipeline = [
 test_pipeline = [
     dict(
         type='LoadPointsFromFile',
+        file_client_args=file_client_args,
         coord_type='DEPTH',
         shift_height=False,
         use_color=True,
@@ -76,6 +91,7 @@ test_pipeline = [
 eval_pipeline = [
     dict(
         type='LoadPointsFromFile',
+        file_client_args=file_client_args,
         coord_type='DEPTH',
         shift_height=False,
         use_color=True,
@@ -83,6 +99,7 @@ eval_pipeline = [
         use_dim=[0, 1, 2, 3, 4, 5]),
     dict(
         type='LoadAnnotations3D',
+        file_client_args=file_client_args,
         with_bbox_3d=False,
         with_label_3d=False,
         with_mask_3d=False,
@@ -116,7 +133,8 @@ data = dict(
         scene_idxs=[
             data_root + f'seg_info/Area_{i}_resampled_scene_idxs.npy'
             for i in train_area
-        ]),
+        ],
+        file_client_args=file_client_args),
     val=dict(
         type=dataset_type,
         data_root=data_root,
@@ -126,7 +144,8 @@ data = dict(
         test_mode=True,
         ignore_index=len(class_names),
         scene_idxs=data_root +
-        f'seg_info/Area_{test_area}_resampled_scene_idxs.npy'),
+        f'seg_info/Area_{test_area}_resampled_scene_idxs.npy',
+        file_client_args=file_client_args),
     test=dict(
         type=dataset_type,
         data_root=data_root,
@@ -134,6 +153,7 @@ data = dict(
         pipeline=test_pipeline,
         classes=class_names,
         test_mode=True,
-        ignore_index=len(class_names)))
+        ignore_index=len(class_names),
+        file_client_args=file_client_args))
 
 evaluation = dict(pipeline=eval_pipeline)

mmdet3d/datasets/custom_3d_seg.py

@@ -276,7 +276,8 @@ class Custom3DSegDataset(Dataset):
         if scene_idxs is None:
             scene_idxs = np.arange(len(self.data_infos))
         if isinstance(scene_idxs, str):
-            scene_idxs = np.load(scene_idxs)
+            with self.file_client.get_local_path(scene_idxs) as local_path:
+                scene_idxs = np.load(local_path)
         else:
             scene_idxs = np.array(scene_idxs)
 

mmdet3d/datasets/s3dis_dataset.py

@@ -381,7 +381,8 @@ class S3DISSegDataset(_S3DISSegDataset):
                 modality=modality,
                 test_mode=test_mode,
                 ignore_index=ignore_index,
-                scene_idxs=scene_idxs[i]) for i in range(len(ann_files))
+                scene_idxs=scene_idxs[i],
+                **kwargs) for i in range(len(ann_files))
         ]
 
         # data_infos and scene_idxs need to be concat