loading.py 24.1 KB
Newer Older
zhangwenwei's avatar
zhangwenwei committed
1
2
3
import mmcv
import numpy as np

4
from mmdet3d.core.points import BasePoints, get_points_type
5
from mmdet.datasets.builder import PIPELINES
6
from mmdet.datasets.pipelines import LoadAnnotations, LoadImageFromFile
zhangwenwei's avatar
zhangwenwei committed
7
8


9
@PIPELINES.register_module()
zhangwenwei's avatar
zhangwenwei committed
10
class LoadMultiViewImageFromFiles(object):
zhangwenwei's avatar
zhangwenwei committed
11
    """Load multi channel images from a list of separate channel files.
zhangwenwei's avatar
zhangwenwei committed
12

liyinhao's avatar
liyinhao committed
13
14
15
16
17
18
    Expects results['img_filename'] to be a list of filenames.

    Args:
        to_float32 (bool): Whether to convert the img to float32.
            Defaults to False.
        color_type (str): Color type of the file. Defaults to 'unchanged'.
zhangwenwei's avatar
zhangwenwei committed
19
    """
zhangwenwei's avatar
zhangwenwei committed
20

zhangwenwei's avatar
zhangwenwei committed
21
22
23
    def __init__(self, to_float32=False, color_type='unchanged'):
        self.to_float32 = to_float32
        self.color_type = color_type
zhangwenwei's avatar
zhangwenwei committed
24
25

    def __call__(self, results):
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
        """Call function to load multi-view image from files.

        Args:
            results (dict): Result dict containing multi-view image filenames.

        Returns:
            dict: The result dict containing the multi-view image data. \
                Added keys and values are described below.

                - filename (str): Multi-view image filenames.
                - img (np.ndarray): Multi-view image arrays.
                - img_shape (tuple[int]): Shape of multi-view image arrays.
                - ori_shape (tuple[int]): Shape of original image arrays.
                - pad_shape (tuple[int]): Shape of padded image arrays.
                - scale_factor (float): Scale factor.
                - img_norm_cfg (dict): Normalization configuration of images.
        """
zhangwenwei's avatar
zhangwenwei committed
43
        filename = results['img_filename']
zhangwenwei's avatar
zhangwenwei committed
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
        img = np.stack(
            [mmcv.imread(name, self.color_type) for name in filename], axis=-1)
        if self.to_float32:
            img = img.astype(np.float32)
        results['filename'] = filename
        results['img'] = img
        results['img_shape'] = img.shape
        results['ori_shape'] = img.shape
        # Set initial values for default meta_keys
        results['pad_shape'] = img.shape
        results['scale_factor'] = 1.0
        num_channels = 1 if len(img.shape) < 3 else img.shape[2]
        results['img_norm_cfg'] = dict(
            mean=np.zeros(num_channels, dtype=np.float32),
            std=np.ones(num_channels, dtype=np.float32),
            to_rgb=False)
zhangwenwei's avatar
zhangwenwei committed
60
61
62
        return results

    def __repr__(self):
63
        """str: Return a string that describes the module."""
64
65
        return f'{self.__class__.__name__} (to_float32={self.to_float32}, '\
            f"color_type='{self.color_type}')"
zhangwenwei's avatar
zhangwenwei committed
66
67


68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
@PIPELINES.register_module()
class LoadImageFromFileMono3D(LoadImageFromFile):
    """Load an image from file in monocular 3D object detection. Compared to 2D
    detection, additional camera parameters need to be loaded.

    Args:
        kwargs (dict): Arguments are the same as those in \
            :class:`LoadImageFromFile`.
    """

    def __call__(self, results):
        """Call functions to load image and get image meta information.

        Args:
            results (dict): Result dict from :obj:`mmdet.CustomDataset`.

        Returns:
            dict: The dict contains loaded image and meta information.
        """
        super().__call__(results)
        results['cam_intrinsic'] = results['img_info']['cam_intrinsic']
        return results


zhangwenwei's avatar
zhangwenwei committed
92
93
@PIPELINES.register_module()
class LoadPointsFromMultiSweeps(object):
zhangwenwei's avatar
zhangwenwei committed
94
    """Load points from multiple sweeps.
zhangwenwei's avatar
zhangwenwei committed
95

zhangwenwei's avatar
zhangwenwei committed
96
97
98
    This is usually used for nuScenes dataset to utilize previous sweeps.

    Args:
99
100
101
        sweeps_num (int): Number of sweeps. Defaults to 10.
        load_dim (int): Dimension number of the loaded points. Defaults to 5.
        use_dim (list[int]): Which dimension to use. Defaults to [0, 1, 2, 4].
zhangwenwei's avatar
zhangwenwei committed
102
103
        file_client_args (dict): Config dict of file clients, refer to
            https://github.com/open-mmlab/mmcv/blob/master/mmcv/fileio/file_client.py
liyinhao's avatar
liyinhao committed
104
            for more details. Defaults to dict(backend='disk').
105
106
107
108
109
110
111
        pad_empty_sweeps (bool): Whether to repeat keyframe when
            sweeps is empty. Defaults to False.
        remove_close (bool): Whether to remove close points.
            Defaults to False.
        test_mode (bool): If test_model=True used for testing, it will not
            randomly sample sweeps but select the nearest N frames.
            Defaults to False.
zhangwenwei's avatar
zhangwenwei committed
112
113
114
115
116
    """

    def __init__(self,
                 sweeps_num=10,
                 load_dim=5,
117
118
119
120
121
                 use_dim=[0, 1, 2, 4],
                 file_client_args=dict(backend='disk'),
                 pad_empty_sweeps=False,
                 remove_close=False,
                 test_mode=False):
zhangwenwei's avatar
zhangwenwei committed
122
        self.load_dim = load_dim
zhangwenwei's avatar
zhangwenwei committed
123
        self.sweeps_num = sweeps_num
124
        self.use_dim = use_dim
zhangwenwei's avatar
zhangwenwei committed
125
126
        self.file_client_args = file_client_args.copy()
        self.file_client = None
127
128
129
        self.pad_empty_sweeps = pad_empty_sweeps
        self.remove_close = remove_close
        self.test_mode = test_mode
zhangwenwei's avatar
zhangwenwei committed
130
131

    def _load_points(self, pts_filename):
132
133
134
135
136
137
138
139
        """Private function to load point clouds data.

        Args:
            pts_filename (str): Filename of point clouds data.

        Returns:
            np.ndarray: An array containing point clouds data.
        """
zhangwenwei's avatar
zhangwenwei committed
140
141
142
143
144
145
146
147
148
149
150
151
        if self.file_client is None:
            self.file_client = mmcv.FileClient(**self.file_client_args)
        try:
            pts_bytes = self.file_client.get(pts_filename)
            points = np.frombuffer(pts_bytes, dtype=np.float32)
        except ConnectionError:
            mmcv.check_file_exist(pts_filename)
            if pts_filename.endswith('.npy'):
                points = np.load(pts_filename)
            else:
                points = np.fromfile(pts_filename, dtype=np.float32)
        return points
zhangwenwei's avatar
zhangwenwei committed
152

153
154
155
156
157
158
159
160
161
162
163
    def _remove_close(self, points, radius=1.0):
        """Removes point too close within a certain radius from origin.

        Args:
            points (np.ndarray): Sweep points.
            radius (float): Radius below which points are removed.
                Defaults to 1.0.

        Returns:
            np.ndarray: Points after removing.
        """
164
165
166
167
168
169
170
171
        if isinstance(points, np.ndarray):
            points_numpy = points
        elif isinstance(points, BasePoints):
            points_numpy = points.tensor.numpy()
        else:
            raise NotImplementedError
        x_filt = np.abs(points_numpy[:, 0]) < radius
        y_filt = np.abs(points_numpy[:, 1]) < radius
172
        not_close = np.logical_not(np.logical_and(x_filt, y_filt))
173
        return points[not_close]
174

zhangwenwei's avatar
zhangwenwei committed
175
    def __call__(self, results):
176
177
178
179
180
181
182
183
184
185
186
187
        """Call function to load multi-sweep point clouds from files.

        Args:
            results (dict): Result dict containing multi-sweep point cloud \
                filenames.

        Returns:
            dict: The result dict containing the multi-sweep points data. \
                Added key and value are described below.

                - points (np.ndarray): Multi-sweep point cloud arrays.
        """
zhangwenwei's avatar
zhangwenwei committed
188
        points = results['points']
189
        points.tensor[:, 4] = 0
zhangwenwei's avatar
zhangwenwei committed
190
191
        sweep_points_list = [points]
        ts = results['timestamp']
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
        if self.pad_empty_sweeps and len(results['sweeps']) == 0:
            for i in range(self.sweeps_num):
                if self.remove_close:
                    sweep_points_list.append(self._remove_close(points))
                else:
                    sweep_points_list.append(points)
        else:
            if len(results['sweeps']) <= self.sweeps_num:
                choices = np.arange(len(results['sweeps']))
            elif self.test_mode:
                choices = np.arange(self.sweeps_num)
            else:
                choices = np.random.choice(
                    len(results['sweeps']), self.sweeps_num, replace=False)
            for idx in choices:
                sweep = results['sweeps'][idx]
                points_sweep = self._load_points(sweep['data_path'])
                points_sweep = np.copy(points_sweep).reshape(-1, self.load_dim)
                if self.remove_close:
                    points_sweep = self._remove_close(points_sweep)
                sweep_ts = sweep['timestamp'] / 1e6
                points_sweep[:, :3] = points_sweep[:, :3] @ sweep[
                    'sensor2lidar_rotation'].T
                points_sweep[:, :3] += sweep['sensor2lidar_translation']
                points_sweep[:, 4] = ts - sweep_ts
217
                points_sweep = points.new_point(points_sweep)
218
219
                sweep_points_list.append(points_sweep)

220
221
        points = points.cat(sweep_points_list)
        points = points[:, self.use_dim]
zhangwenwei's avatar
zhangwenwei committed
222
223
224
225
        results['points'] = points
        return results

    def __repr__(self):
226
        """str: Return a string that describes the module."""
zhangwenwei's avatar
zhangwenwei committed
227
        return f'{self.__class__.__name__}(sweeps_num={self.sweeps_num})'
wuyuefeng's avatar
wuyuefeng committed
228
229
230
231
232
233
234
235
236
237


@PIPELINES.register_module()
class PointSegClassMapping(object):
    """Map original semantic class to valid category ids.

    Map valid classes as 0~len(valid_cat_ids)-1 and
    others as len(valid_cat_ids).

    Args:
238
        valid_cat_ids (tuple[int]): A tuple of valid category.
wuyuefeng's avatar
wuyuefeng committed
239
240
241
242
243
244
    """

    def __init__(self, valid_cat_ids):
        self.valid_cat_ids = valid_cat_ids

    def __call__(self, results):
245
246
247
248
249
250
251
252
253
254
255
        """Call function to map original semantic class to valid category ids.

        Args:
            results (dict): Result dict containing point semantic masks.

        Returns:
            dict: The result dict containing the mapped category ids. \
                Updated key and value are described below.

                - pts_semantic_mask (np.ndarray): Mapped semantic masks.
        """
wuyuefeng's avatar
wuyuefeng committed
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
        assert 'pts_semantic_mask' in results
        pts_semantic_mask = results['pts_semantic_mask']
        neg_cls = len(self.valid_cat_ids)

        for i in range(pts_semantic_mask.shape[0]):
            if pts_semantic_mask[i] in self.valid_cat_ids:
                converted_id = self.valid_cat_ids.index(pts_semantic_mask[i])
                pts_semantic_mask[i] = converted_id
            else:
                pts_semantic_mask[i] = neg_cls

        results['pts_semantic_mask'] = pts_semantic_mask
        return results

    def __repr__(self):
271
        """str: Return a string that describes the module."""
wuyuefeng's avatar
wuyuefeng committed
272
        repr_str = self.__class__.__name__
273
        repr_str += f'(valid_cat_ids={self.valid_cat_ids})'
wuyuefeng's avatar
wuyuefeng committed
274
275
276
277
278
        return repr_str


@PIPELINES.register_module()
class NormalizePointsColor(object):
zhangwenwei's avatar
zhangwenwei committed
279
    """Normalize color of points.
wuyuefeng's avatar
wuyuefeng committed
280
281
282
283
284
285
286
287
288

    Args:
        color_mean (list[float]): Mean color of the point cloud.
    """

    def __init__(self, color_mean):
        self.color_mean = color_mean

    def __call__(self, results):
289
290
291
292
293
294
295
296
297
298
299
        """Call function to normalize color of points.

        Args:
            results (dict): Result dict containing point clouds data.

        Returns:
            dict: The result dict containing the normalized points. \
                Updated key and value are described below.

                - points (np.ndarray): Points after color normalization.
        """
wuyuefeng's avatar
wuyuefeng committed
300
        points = results['points']
301
302
303
304
305
306
307
        assert points.attribute_dims is not None and \
            'color' in points.attribute_dims.keys(), \
            'Expect points have color attribute'
        if self.color_mean is not None:
            points.color = points.color - \
                points.color.new_tensor(self.color_mean)
        points.color = points.color / 255.0
wuyuefeng's avatar
wuyuefeng committed
308
309
310
311
        results['points'] = points
        return results

    def __repr__(self):
312
        """str: Return a string that describes the module."""
wuyuefeng's avatar
wuyuefeng committed
313
        repr_str = self.__class__.__name__
314
        repr_str += f'(color_mean={self.color_mean})'
wuyuefeng's avatar
wuyuefeng committed
315
316
317
318
319
320
321
322
323
324
        return repr_str


@PIPELINES.register_module()
class LoadPointsFromFile(object):
    """Load Points From File.

    Load sunrgbd and scannet points from file.

    Args:
325
326
327
328
329
        coord_type (str): The type of coordinates of points cloud.
            Available options includes:
            - 'LIDAR': Points in LiDAR coordinates.
            - 'DEPTH': Points in depth coordinates, usually for indoor dataset.
            - 'CAMERA': Points in camera coordinates.
330
331
        load_dim (int): The dimension of the loaded points.
            Defaults to 6.
wuyuefeng's avatar
wuyuefeng committed
332
        use_dim (list[int]): Which dimensions of the points to be used.
liyinhao's avatar
liyinhao committed
333
334
335
            Defaults to [0, 1, 2]. For KITTI dataset, set use_dim=4
            or use_dim=[0, 1, 2, 3] to use the intensity dimension.
        shift_height (bool): Whether to use shifted height. Defaults to False.
336
        use_color (bool): Whether to use color features. Defaults to False.
wuyuefeng's avatar
wuyuefeng committed
337
338
        file_client_args (dict): Config dict of file clients, refer to
            https://github.com/open-mmlab/mmcv/blob/master/mmcv/fileio/file_client.py
liyinhao's avatar
liyinhao committed
339
            for more details. Defaults to dict(backend='disk').
wuyuefeng's avatar
wuyuefeng committed
340
341
342
    """

    def __init__(self,
343
                 coord_type,
wuyuefeng's avatar
wuyuefeng committed
344
345
346
                 load_dim=6,
                 use_dim=[0, 1, 2],
                 shift_height=False,
347
                 use_color=False,
wuyuefeng's avatar
wuyuefeng committed
348
349
                 file_client_args=dict(backend='disk')):
        self.shift_height = shift_height
350
        self.use_color = use_color
wuyuefeng's avatar
wuyuefeng committed
351
352
353
354
        if isinstance(use_dim, int):
            use_dim = list(range(use_dim))
        assert max(use_dim) < load_dim, \
            f'Expect all used dimensions < {load_dim}, got {use_dim}'
355
        assert coord_type in ['CAMERA', 'LIDAR', 'DEPTH']
wuyuefeng's avatar
wuyuefeng committed
356

357
        self.coord_type = coord_type
wuyuefeng's avatar
wuyuefeng committed
358
359
360
361
362
363
        self.load_dim = load_dim
        self.use_dim = use_dim
        self.file_client_args = file_client_args.copy()
        self.file_client = None

    def _load_points(self, pts_filename):
364
365
366
367
368
369
370
371
        """Private function to load point clouds data.

        Args:
            pts_filename (str): Filename of point clouds data.

        Returns:
            np.ndarray: An array containing point clouds data.
        """
wuyuefeng's avatar
wuyuefeng committed
372
373
374
375
376
377
378
379
380
381
382
        if self.file_client is None:
            self.file_client = mmcv.FileClient(**self.file_client_args)
        try:
            pts_bytes = self.file_client.get(pts_filename)
            points = np.frombuffer(pts_bytes, dtype=np.float32)
        except ConnectionError:
            mmcv.check_file_exist(pts_filename)
            if pts_filename.endswith('.npy'):
                points = np.load(pts_filename)
            else:
                points = np.fromfile(pts_filename, dtype=np.float32)
383

wuyuefeng's avatar
wuyuefeng committed
384
385
386
        return points

    def __call__(self, results):
387
388
389
390
391
392
393
394
395
396
397
        """Call function to load points data from file.

        Args:
            results (dict): Result dict containing point clouds data.

        Returns:
            dict: The result dict containing the point clouds data. \
                Added key and value are described below.

                - points (np.ndarray): Point clouds data.
        """
wuyuefeng's avatar
wuyuefeng committed
398
399
400
401
        pts_filename = results['pts_filename']
        points = self._load_points(pts_filename)
        points = points.reshape(-1, self.load_dim)
        points = points[:, self.use_dim]
402
        attribute_dims = None
wuyuefeng's avatar
wuyuefeng committed
403
404
405
406

        if self.shift_height:
            floor_height = np.percentile(points[:, 2], 0.99)
            height = points[:, 2] - floor_height
407
408
409
            points = np.concatenate(
                [points[:, :3],
                 np.expand_dims(height, 1), points[:, 3:]], 1)
410
411
            attribute_dims = dict(height=3)

412
413
414
415
416
417
418
419
420
421
422
        if self.use_color:
            assert len(self.use_dim) >= 6
            if attribute_dims is None:
                attribute_dims = dict()
            attribute_dims.update(
                dict(color=[
                    points.shape[1] - 3,
                    points.shape[1] - 2,
                    points.shape[1] - 1,
                ]))

423
424
425
        points_class = get_points_type(self.coord_type)
        points = points_class(
            points, points_dim=points.shape[-1], attribute_dims=attribute_dims)
wuyuefeng's avatar
wuyuefeng committed
426
        results['points'] = points
427

wuyuefeng's avatar
wuyuefeng committed
428
429
430
        return results

    def __repr__(self):
431
        """str: Return a string that describes the module."""
liyinhao's avatar
liyinhao committed
432
        repr_str = self.__class__.__name__ + '('
433
434
435
436
437
        repr_str += f'shift_height={self.shift_height}, '
        repr_str += f'use_color={self.use_color}, '
        repr_str += f'file_client_args={self.file_client_args}, '
        repr_str += f'load_dim={self.load_dim}, '
        repr_str += f'use_dim={self.use_dim})'
wuyuefeng's avatar
wuyuefeng committed
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
        return repr_str


@PIPELINES.register_module()
class LoadAnnotations3D(LoadAnnotations):
    """Load Annotations3D.

    Load instance mask and semantic mask of points and
    encapsulate the items into related fields.

    Args:
        with_bbox_3d (bool, optional): Whether to load 3D boxes.
            Defaults to True.
        with_label_3d (bool, optional): Whether to load 3D labels.
            Defaults to True.
453
454
        with_attr_label (bool, optional): Whether to load attribute label.
            Defaults to False.
wuyuefeng's avatar
wuyuefeng committed
455
456
457
458
459
460
461
462
463
464
465
466
        with_mask_3d (bool, optional): Whether to load 3D instance masks.
            for points. Defaults to False.
        with_seg_3d (bool, optional): Whether to load 3D semantic masks.
            for points. Defaults to False.
        with_bbox (bool, optional): Whether to load 2D boxes.
            Defaults to False.
        with_label (bool, optional): Whether to load 2D labels.
            Defaults to False.
        with_mask (bool, optional): Whether to load 2D instance masks.
            Defaults to False.
        with_seg (bool, optional): Whether to load 2D semantic masks.
            Defaults to False.
467
468
        with_bbox_depth (bool, optional): Whether to load 2.5D boxes.
            Defaults to False.
wuyuefeng's avatar
wuyuefeng committed
469
470
        poly2mask (bool, optional): Whether to convert polygon annotations
            to bitmasks. Defaults to True.
471
472
        seg_3d_dtype (dtype, optional): Dtype of 3D semantic masks.
            Defaults to int64
wuyuefeng's avatar
wuyuefeng committed
473
474
475
476
477
478
479
480
        file_client_args (dict): Config dict of file clients, refer to
            https://github.com/open-mmlab/mmcv/blob/master/mmcv/fileio/file_client.py
            for more details.
    """

    def __init__(self,
                 with_bbox_3d=True,
                 with_label_3d=True,
481
                 with_attr_label=False,
wuyuefeng's avatar
wuyuefeng committed
482
483
484
485
486
487
                 with_mask_3d=False,
                 with_seg_3d=False,
                 with_bbox=False,
                 with_label=False,
                 with_mask=False,
                 with_seg=False,
488
                 with_bbox_depth=False,
wuyuefeng's avatar
wuyuefeng committed
489
                 poly2mask=True,
490
                 seg_3d_dtype='int',
wuyuefeng's avatar
wuyuefeng committed
491
492
493
494
495
496
497
498
499
                 file_client_args=dict(backend='disk')):
        super().__init__(
            with_bbox,
            with_label,
            with_mask,
            with_seg,
            poly2mask,
            file_client_args=file_client_args)
        self.with_bbox_3d = with_bbox_3d
500
        self.with_bbox_depth = with_bbox_depth
wuyuefeng's avatar
wuyuefeng committed
501
        self.with_label_3d = with_label_3d
502
        self.with_attr_label = with_attr_label
wuyuefeng's avatar
wuyuefeng committed
503
504
        self.with_mask_3d = with_mask_3d
        self.with_seg_3d = with_seg_3d
505
        self.seg_3d_dtype = seg_3d_dtype
wuyuefeng's avatar
wuyuefeng committed
506
507

    def _load_bboxes_3d(self, results):
508
509
510
511
512
513
514
515
        """Private function to load 3D bounding box annotations.

        Args:
            results (dict): Result dict from :obj:`mmdet3d.CustomDataset`.

        Returns:
            dict: The dict containing loaded 3D bounding box annotations.
        """
wuyuefeng's avatar
wuyuefeng committed
516
517
518
519
        results['gt_bboxes_3d'] = results['ann_info']['gt_bboxes_3d']
        results['bbox3d_fields'].append('gt_bboxes_3d')
        return results

520
521
522
523
524
525
526
527
528
529
530
531
532
    def _load_bboxes_depth(self, results):
        """Private function to load 2.5D bounding box annotations.

        Args:
            results (dict): Result dict from :obj:`mmdet3d.CustomDataset`.

        Returns:
            dict: The dict containing loaded 2.5D bounding box annotations.
        """
        results['centers2d'] = results['ann_info']['centers2d']
        results['depths'] = results['ann_info']['depths']
        return results

wuyuefeng's avatar
wuyuefeng committed
533
    def _load_labels_3d(self, results):
534
535
536
537
538
539
540
541
        """Private function to load label annotations.

        Args:
            results (dict): Result dict from :obj:`mmdet3d.CustomDataset`.

        Returns:
            dict: The dict containing loaded label annotations.
        """
wuyuefeng's avatar
wuyuefeng committed
542
543
544
        results['gt_labels_3d'] = results['ann_info']['gt_labels_3d']
        return results

545
546
547
548
549
550
551
552
553
554
555
556
    def _load_attr_labels(self, results):
        """Private function to load label annotations.

        Args:
            results (dict): Result dict from :obj:`mmdet3d.CustomDataset`.

        Returns:
            dict: The dict containing loaded label annotations.
        """
        results['attr_labels'] = results['ann_info']['attr_labels']
        return results

wuyuefeng's avatar
wuyuefeng committed
557
    def _load_masks_3d(self, results):
558
559
560
561
562
563
564
565
        """Private function to load 3D mask annotations.

        Args:
            results (dict): Result dict from :obj:`mmdet3d.CustomDataset`.

        Returns:
            dict: The dict containing loaded 3D mask annotations.
        """
wuyuefeng's avatar
wuyuefeng committed
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
        pts_instance_mask_path = results['ann_info']['pts_instance_mask_path']

        if self.file_client is None:
            self.file_client = mmcv.FileClient(**self.file_client_args)
        try:
            mask_bytes = self.file_client.get(pts_instance_mask_path)
            pts_instance_mask = np.frombuffer(mask_bytes, dtype=np.int)
        except ConnectionError:
            mmcv.check_file_exist(pts_instance_mask_path)
            pts_instance_mask = np.fromfile(
                pts_instance_mask_path, dtype=np.long)

        results['pts_instance_mask'] = pts_instance_mask
        results['pts_mask_fields'].append('pts_instance_mask')
        return results

    def _load_semantic_seg_3d(self, results):
583
584
585
586
587
588
589
590
        """Private function to load 3D semantic segmentation annotations.

        Args:
            results (dict): Result dict from :obj:`mmdet3d.CustomDataset`.

        Returns:
            dict: The dict containing the semantic segmentation annotations.
        """
wuyuefeng's avatar
wuyuefeng committed
591
592
593
594
595
596
597
        pts_semantic_mask_path = results['ann_info']['pts_semantic_mask_path']

        if self.file_client is None:
            self.file_client = mmcv.FileClient(**self.file_client_args)
        try:
            mask_bytes = self.file_client.get(pts_semantic_mask_path)
            # add .copy() to fix read-only bug
598
599
            pts_semantic_mask = np.frombuffer(
                mask_bytes, dtype=self.seg_3d_dtype).copy()
wuyuefeng's avatar
wuyuefeng committed
600
601
602
603
604
605
606
607
608
609
        except ConnectionError:
            mmcv.check_file_exist(pts_semantic_mask_path)
            pts_semantic_mask = np.fromfile(
                pts_semantic_mask_path, dtype=np.long)

        results['pts_semantic_mask'] = pts_semantic_mask
        results['pts_seg_fields'].append('pts_semantic_mask')
        return results

    def __call__(self, results):
610
611
612
613
614
615
616
617
618
        """Call function to load multiple types annotations.

        Args:
            results (dict): Result dict from :obj:`mmdet3d.CustomDataset`.

        Returns:
            dict: The dict containing loaded 3D bounding box, label, mask and
                semantic segmentation annotations.
        """
wuyuefeng's avatar
wuyuefeng committed
619
620
621
622
623
        results = super().__call__(results)
        if self.with_bbox_3d:
            results = self._load_bboxes_3d(results)
            if results is None:
                return None
624
625
626
627
        if self.with_bbox_depth:
            results = self._load_bboxes_depth(results)
            if results is None:
                return None
wuyuefeng's avatar
wuyuefeng committed
628
629
        if self.with_label_3d:
            results = self._load_labels_3d(results)
630
631
        if self.with_attr_label:
            results = self._load_attr_labels(results)
wuyuefeng's avatar
wuyuefeng committed
632
633
634
635
636
637
638
639
        if self.with_mask_3d:
            results = self._load_masks_3d(results)
        if self.with_seg_3d:
            results = self._load_semantic_seg_3d(results)

        return results

    def __repr__(self):
640
        """str: Return a string that describes the module."""
wuyuefeng's avatar
wuyuefeng committed
641
642
        indent_str = '    '
        repr_str = self.__class__.__name__ + '(\n'
liyinhao's avatar
liyinhao committed
643
644
        repr_str += f'{indent_str}with_bbox_3d={self.with_bbox_3d}, '
        repr_str += f'{indent_str}with_label_3d={self.with_label_3d}, '
645
        repr_str += f'{indent_str}with_attr_label={self.with_attr_label}, '
liyinhao's avatar
liyinhao committed
646
647
648
649
650
651
        repr_str += f'{indent_str}with_mask_3d={self.with_mask_3d}, '
        repr_str += f'{indent_str}with_seg_3d={self.with_seg_3d}, '
        repr_str += f'{indent_str}with_bbox={self.with_bbox}, '
        repr_str += f'{indent_str}with_label={self.with_label}, '
        repr_str += f'{indent_str}with_mask={self.with_mask}, '
        repr_str += f'{indent_str}with_seg={self.with_seg}, '
652
        repr_str += f'{indent_str}with_bbox_depth={self.with_bbox_depth}, '
wuyuefeng's avatar
wuyuefeng committed
653
654
        repr_str += f'{indent_str}poly2mask={self.poly2mask})'
        return repr_str