[Refactor]Refactor nus dataset

configs/_base_/datasets/nus-3d.py

@@ -6,17 +6,14 @@ class_names = [
     'car', 'truck', 'trailer', 'bus', 'construction_vehicle', 'bicycle',
     'motorcycle', 'pedestrian', 'traffic_cone', 'barrier'
 ]
+metainfo = dict(CLASSES=class_names)
 dataset_type = 'NuScenesDataset'
 data_root = 'data/nuscenes/'
 # Input modality for nuScenes dataset, this is consistent with the submission
 # format which requires the information in input_modality.
-input_modality = dict(
-    use_lidar=True,
-    use_camera=False,
-    use_radar=False,
-    use_map=False,
-    use_external=False)
+input_modality = dict(use_lidar=True, use_camera=False)
 file_client_args = dict(backend='disk')
+data_prefix = dict(pts='samples/LIDAR_TOP', img='')
 # Uncomment the following if use ceph or other file clients.
 # See https://mmcv.readthedocs.io/en/latest/api.html#mmcv.fileio.FileClient
 # for more details.
@@ -48,8 +45,9 @@ train_pipeline = [
     dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range),
     dict(type='ObjectNameFilter', classes=class_names),
     dict(type='PointShuffle'),
-    dict(type='DefaultFormatBundle3D', class_names=class_names),
-    dict(type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d'])
+    dict(
+        type='Pack3DDetInputs',
+        keys=['points', 'gt_bboxes_3d', 'gt_labels_3d'])
 ]
 test_pipeline = [
     dict(
@@ -61,6 +59,7 @@ test_pipeline = [
     dict(
         type='LoadPointsFromMultiSweeps',
         sweeps_num=10,
+        test_mode=True,
         file_client_args=file_client_args),
     dict(
         type='MultiScaleFlipAug3D',
@@ -75,13 +74,9 @@ test_pipeline = [
                 translation_std=[0, 0, 0]),
             dict(type='RandomFlip3D'),
             dict(
-                type='PointsRangeFilter', point_cloud_range=point_cloud_range),
-            dict(
-                type='DefaultFormatBundle3D',
-                class_names=class_names,
-                with_label=False),
-            dict(type='Collect3D', keys=['points'])
-        ])
+                type='PointsRangeFilter', point_cloud_range=point_cloud_range)
+        ]),
+    dict(type='Pack3DDetInputs', keys=['points'])
 ]
 # construct a pipeline for data and gt loading in show function
 # please keep its loading function consistent with test_pipeline (e.g. client)
@@ -95,48 +90,63 @@ eval_pipeline = [
     dict(
         type='LoadPointsFromMultiSweeps',
         sweeps_num=10,
+        test_mode=True,
         file_client_args=file_client_args),
-    dict(
-        type='DefaultFormatBundle3D',
-        class_names=class_names,
-        with_label=False),
-    dict(type='Collect3D', keys=['points'])
+    dict(type='Pack3DDetInputs', keys=['points'])
 ]
-
-data = dict(
-    samples_per_gpu=4,
-    workers_per_gpu=4,
-    train=dict(
+train_dataloader = dict(
+    batch_size=4,
+    num_workers=4,
+    persistent_workers=True,
+    sampler=dict(type='DefaultSampler', shuffle=True),
+    dataset=dict(
         type=dataset_type,
         data_root=data_root,
-        ann_file=data_root + 'nuscenes_infos_train.pkl',
+        ann_file='nuscenes_infos_train.pkl',
         pipeline=train_pipeline,
-        classes=class_names,
+        metainfo=metainfo,
         modality=input_modality,
         test_mode=False,
+        data_prefix=data_prefix,
         # we use box_type_3d='LiDAR' in kitti and nuscenes dataset
         # and box_type_3d='Depth' in sunrgbd and scannet dataset.
-        box_type_3d='LiDAR'),
-    val=dict(
+        box_type_3d='LiDAR'))
+test_dataloader = dict(
+    batch_size=1,
+    num_workers=1,
+    persistent_workers=True,
+    drop_last=False,
+    sampler=dict(type='DefaultSampler', shuffle=False),
+    dataset=dict(
         type=dataset_type,
         data_root=data_root,
-        ann_file=data_root + 'nuscenes_infos_val.pkl',
+        ann_file='nuscenes_infos_val.pkl',
         pipeline=test_pipeline,
-        classes=class_names,
+        metainfo=metainfo,
         modality=input_modality,
+        data_prefix=data_prefix,
         test_mode=True,
-        box_type_3d='LiDAR'),
-    test=dict(
+        box_type_3d='LiDAR'))
+val_dataloader = dict(
+    batch_size=1,
+    num_workers=1,
+    persistent_workers=True,
+    drop_last=False,
+    sampler=dict(type='DefaultSampler', shuffle=False),
+    dataset=dict(
         type=dataset_type,
         data_root=data_root,
-        ann_file=data_root + 'nuscenes_infos_val.pkl',
+        ann_file='nuscenes_infos_val.pkl',
         pipeline=test_pipeline,
-        classes=class_names,
+        metainfo=metainfo,
         modality=input_modality,
         test_mode=True,
+        data_prefix=data_prefix,
         box_type_3d='LiDAR'))
-# For nuScenes dataset, we usually evaluate the model at the end of training.
-# Since the models are trained by 24 epochs by default, we set evaluation
-# interval to be 24. Please change the interval accordingly if you do not
-# use a default schedule.
-evaluation = dict(interval=24, pipeline=eval_pipeline)
+
+val_evaluator = dict(
+    type='NuScenesMetric',
+    data_root=data_root,
+    ann_file=data_root + 'nuscenes_infos_val.pkl',
+    metric='bbox')
+test_evaluator = val_evaluator

configs/_base_/models/hv_pointpillars_fpn_nus.py

@@ -6,6 +6,7 @@
 voxel_size = [0.25, 0.25, 8]
 model = dict(
     type='MVXFasterRCNN',
+    data_preprocessor=dict(type='Det3DDataPreprocessor'),
     pts_voxel_layer=dict(
         max_num_points=64,
         point_cloud_range=[-50, -50, -5, 50, 50, 3],
@@ -62,19 +63,21 @@ model = dict(
         dir_offset=-0.7854,  # -pi / 4
         bbox_coder=dict(type='DeltaXYZWLHRBBoxCoder', code_size=9),
         loss_cls=dict(
-            type='FocalLoss',
+            type='mmdet.FocalLoss',
             use_sigmoid=True,
             gamma=2.0,
             alpha=0.25,
             loss_weight=1.0),
-        loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0),
+        loss_bbox=dict(
+            type='mmdet.SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0),
         loss_dir=dict(
-            type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.2)),
+            type='mmdet.CrossEntropyLoss', use_sigmoid=False,
+            loss_weight=0.2)),
     # model training and testing settings
     train_cfg=dict(
         pts=dict(
             assigner=dict(
-                type='MaxIoUAssigner',
+                type='Max3DIoUAssigner',
                 iou_calculator=dict(type='BboxOverlapsNearest3D'),
                 pos_iou_thr=0.6,
                 neg_iou_thr=0.3,

configs/_base_/schedules/schedule_2x.py

 # optimizer
 # This schedule is mainly used by models on nuScenes dataset
-optimizer = dict(type='AdamW', lr=0.001, weight_decay=0.01)
-# max_norm=10 is better for SECOND
-optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2))
-lr_config = dict(
-    policy='step',
-    warmup='linear',
-    warmup_iters=1000,
-    warmup_ratio=1.0 / 1000,
-    step=[20, 23])
-momentum_config = None
-# runtime settings
-runner = dict(type='EpochBasedRunner', max_epochs=24)
+lr = 0.001
+optim_wrapper = dict(
+    type='OptimWrapper',
+    optimizer=dict(type='AdamW', lr=lr, weight_decay=0.01),
+    # max_norm=10 is better for SECOND
+    clip_grad=dict(max_norm=35, norm_type=2))
+
+# training schedule for 2x
+train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=24, val_interval=1)
+val_cfg = dict(type='ValLoop')
+test_cfg = dict(type='TestLoop')
+
+# learning rate
+param_scheduler = [
+    dict(
+        type='LinearLR',
+        start_factor=1.0 / 1000,
+        by_epoch=False,
+        begin=0,
+        end=1000),
+    dict(
+        type='MultiStepLR',
+        begin=0,
+        end=24,
+        by_epoch=True,
+        milestones=[20, 23],
+        gamma=0.1)
+]

configs/pointpillars/hv_pointpillars_fpn_sbn-all_4x8_2x_nus-3d.py

@@ -3,3 +3,9 @@ _base_ = [
     '../_base_/datasets/nus-3d.py', '../_base_/schedules/schedule_2x.py',
     '../_base_/default_runtime.py'
 ]
+
+# For nuScenes dataset, we usually evaluate the model at the end of training.
+# Since the models are trained by 24 epochs by default, we set evaluation
+# interval to be 24. Please change the interval accordingly if you do not
+# use a default schedule.
+train_cfg = dict(val_interval=24)

configs/pointpillars/hv_pointpillars_fpn_sbn-all_fp16_2x8_2x_nus-3d.py

 _base_ = './hv_pointpillars_fpn_sbn-all_4x8_2x_nus-3d.py'
-data = dict(samples_per_gpu=2, workers_per_gpu=2)
+train_dataloader = dict(batch_size=2, num_workers=2)
 # fp16 settings, the loss scale is specifically tuned to avoid Nan
 fp16 = dict(loss_scale=32.)

configs/pointpillars/hv_pointpillars_secfpn_sbn-all_4x8_2x_nus-3d.py

@@ -40,3 +40,9 @@ model = dict(
             custom_values=[0, 0],
             rotations=[0, 1.57],
             reshape_out=True)))
+
+# For nuScenes dataset, we usually evaluate the model at the end of training.
+# Since the models are trained by 24 epochs by default, we set evaluation
+# interval to be 24. Please change the interval accordingly if you do not
+# use a default schedule.
+train_cfg = dict(val_interval=24)

configs/pointpillars/hv_pointpillars_secfpn_sbn-all_fp16_2x8_2x_nus-3d.py

 _base_ = './hv_pointpillars_secfpn_sbn-all_4x8_2x_nus-3d.py'
-data = dict(samples_per_gpu=2, workers_per_gpu=2)
+train_dataloader = dict(batch_size=2, num_workers=2)
 # fp16 settings, the loss scale is specifically tuned to avoid Nan
 fp16 = dict(loss_scale=32.)

mmdet3d/datasets/det3d_dataset.py

@@ -227,8 +227,12 @@ class Det3DDataset(BaseDataset):
         if self.modality['use_camera']:
             for cam_id, img_info in info['images'].items():
                 if 'img_path' in img_info:
-                    img_info['img_path'] = osp.join(
-                        self.data_prefix.get('img', ''), img_info['img_path'])
+                    if cam_id in self.data_prefix:
+                        cam_prefix = self.data_prefix[cam_id]
+                    else:
+                        cam_prefix = self.data_prefix.get('img', '')
+                    img_info['img_path'] = osp.join(cam_prefix,
+                                                    img_info['img_path'])
 
         if not self.test_mode:
             # used in traing

mmdet3d/datasets/nuscenes_dataset.py

 # Copyright (c) OpenMMLab. All rights reserved.
-import tempfile
-from os import path as osp
+from typing import Dict, List
 
-import mmcv
 import numpy as np
-import pyquaternion
-from nuscenes.utils.data_classes import Box as NuScenesBox
 
 from mmdet3d.registry import DATASETS
-from ..core import show_result
-from ..core.bbox import Box3DMode, Coord3DMode, LiDARInstance3DBoxes
+from ..core.bbox import LiDARInstance3DBoxes
 from .det3d_dataset import Det3DDataset
-from .pipelines import Compose
 
 
 @DATASETS.register_module()
@@ -24,263 +18,94 @@ class NuScenesDataset(Det3DDataset):
     for data downloading.
 
     Args:
+        data_root (str): Path of dataset root.
         ann_file (str): Path of annotation file.
         pipeline (list[dict], optional): Pipeline used for data processing.
             Defaults to None.
-        data_root (str): Path of dataset root.
-        classes (tuple[str], optional): Classes used in the dataset.
-            Defaults to None.
-        load_interval (int, optional): Interval of loading the dataset. It is
-            used to uniformly sample the dataset. Defaults to 1.
-        with_velocity (bool, optional): Whether include velocity prediction
-            into the experiments. Defaults to True.
-        modality (dict, optional): Modality to specify the sensor data used
-            as input. Defaults to None.
-        box_type_3d (str, optional): Type of 3D box of this dataset.
+        box_type_3d (str): Type of 3D box of this dataset.
             Based on the `box_type_3d`, the dataset will encapsulate the box
             to its original format then converted them to `box_type_3d`.
             Defaults to 'LiDAR' in this dataset. Available options includes.
+
             - 'LiDAR': Box in LiDAR coordinates.
             - 'Depth': Box in depth coordinates, usually for indoor dataset.
             - 'Camera': Box in camera coordinates.
-        filter_empty_gt (bool, optional): Whether to filter empty GT.
+        modality (dict, optional): Modality to specify the sensor data used
+            as input. Defaults to dict(use_camera=False,use_lidar=True).
+        filter_empty_gt (bool): Whether to filter empty GT.
             Defaults to True.
-        test_mode (bool, optional): Whether the dataset is in test mode.
+        test_mode (bool): Whether the dataset is in test mode.
             Defaults to False.
-        eval_version (bool, optional): Configuration version of evaluation.
-            Defaults to  'detection_cvpr_2019'.
-        use_valid_flag (bool, optional): Whether to use `use_valid_flag` key
+        with_velocity (bool): Whether include velocity prediction
+            into the experiments. Defaults to True.
+        use_valid_flag (bool): Whether to use `use_valid_flag` key
             in the info file as mask to filter gt_boxes and gt_names.
             Defaults to False.
     """
-    NameMapping = {
-        'movable_object.barrier': 'barrier',
-        'vehicle.bicycle': 'bicycle',
-        'vehicle.bus.bendy': 'bus',
-        'vehicle.bus.rigid': 'bus',
-        'vehicle.car': 'car',
-        'vehicle.construction': 'construction_vehicle',
-        'vehicle.motorcycle': 'motorcycle',
-        'human.pedestrian.adult': 'pedestrian',
-        'human.pedestrian.child': 'pedestrian',
-        'human.pedestrian.construction_worker': 'pedestrian',
-        'human.pedestrian.police_officer': 'pedestrian',
-        'movable_object.trafficcone': 'traffic_cone',
-        'vehicle.trailer': 'trailer',
-        'vehicle.truck': 'truck'
-    }
-    DefaultAttribute = {
-        'car': 'vehicle.parked',
-        'pedestrian': 'pedestrian.moving',
-        'trailer': 'vehicle.parked',
-        'truck': 'vehicle.parked',
-        'bus': 'vehicle.moving',
-        'motorcycle': 'cycle.without_rider',
-        'construction_vehicle': 'vehicle.parked',
-        'bicycle': 'cycle.without_rider',
-        'barrier': '',
-        'traffic_cone': '',
-    }
-    AttrMapping = {
-        'cycle.with_rider': 0,
-        'cycle.without_rider': 1,
-        'pedestrian.moving': 2,
-        'pedestrian.standing': 3,
-        'pedestrian.sitting_lying_down': 4,
-        'vehicle.moving': 5,
-        'vehicle.parked': 6,
-        'vehicle.stopped': 7,
+    METAINFO = {
+        'CLASSES':
+        ('car', 'truck', 'trailer', 'bus', 'construction_vehicle', 'bicycle',
+         'motorcycle', 'pedestrian', 'traffic_cone', 'barrier'),
+        'version':
+        'v1.0-trainval'
     }
-    AttrMapping_rev = [
-        'cycle.with_rider',
-        'cycle.without_rider',
-        'pedestrian.moving',
-        'pedestrian.standing',
-        'pedestrian.sitting_lying_down',
-        'vehicle.moving',
-        'vehicle.parked',
-        'vehicle.stopped',
-    ]
-    # https://github.com/nutonomy/nuscenes-devkit/blob/57889ff20678577025326cfc24e57424a829be0a/python-sdk/nuscenes/eval/detection/evaluate.py#L222 # noqa
-    ErrNameMapping = {
-        'trans_err': 'mATE',
-        'scale_err': 'mASE',
-        'orient_err': 'mAOE',
-        'vel_err': 'mAVE',
-        'attr_err': 'mAAE'
-    }
-    CLASSES = ('car', 'truck', 'trailer', 'bus', 'construction_vehicle',
-               'bicycle', 'motorcycle', 'pedestrian', 'traffic_cone',
-               'barrier')
 
     def __init__(self,
-                 ann_file,
-                 pipeline=None,
-                 data_root=None,
-                 classes=None,
-                 load_interval=1,
-                 with_velocity=True,
-                 modality=None,
-                 box_type_3d='LiDAR',
-                 filter_empty_gt=True,
-                 test_mode=False,
-                 eval_version='detection_cvpr_2019',
-                 use_valid_flag=False):
-        self.load_interval = load_interval
+                 data_root: str,
+                 ann_file: str,
+                 pipeline: List[dict] = None,
+                 box_type_3d: str = 'LiDAR',
+                 modality: Dict = dict(
+                     use_camera=False,
+                     use_lidar=True,
+                 ),
+                 filter_empty_gt: bool = True,
+                 test_mode: bool = False,
+                 with_velocity: bool = True,
+                 use_valid_flag: bool = False,
+                 **kwargs):
         self.use_valid_flag = use_valid_flag
+        self.with_velocity = with_velocity
+        assert box_type_3d.lower() == 'lidar'
         super().__init__(
             data_root=data_root,
             ann_file=ann_file,
-            pipeline=pipeline,
-            classes=classes,
             modality=modality,
+            pipeline=pipeline,
             box_type_3d=box_type_3d,
             filter_empty_gt=filter_empty_gt,
-            test_mode=test_mode)
-
-        self.with_velocity = with_velocity
-        self.eval_version = eval_version
-        from nuscenes.eval.detection.config import config_factory
-        self.eval_detection_configs = config_factory(self.eval_version)
-        if self.modality is None:
-            self.modality = dict(
-                use_camera=False,
-                use_lidar=True,
-                use_radar=False,
-                use_map=False,
-                use_external=False,
-            )
-
-    def get_cat_ids(self, idx):
-        """Get category distribution of single scene.
-
-        Args:
-            idx (int): Index of the data_info.
-
-        Returns:
-            dict[list]: for each category, if the current scene
-                contains such boxes, store a list containing idx,
-                otherwise, store empty list.
-        """
-        info = self.data_infos[idx]
-        if self.use_valid_flag:
-            mask = info['valid_flag']
-            gt_names = set(info['gt_names'][mask])
-        else:
-            gt_names = set(info['gt_names'])
-
-        cat_ids = []
-        for name in gt_names:
-            if name in self.CLASSES:
-                cat_ids.append(self.cat2id[name])
-        return cat_ids
+            test_mode=test_mode,
+            **kwargs)
 
-    def load_annotations(self, ann_file):
-        """Load annotations from ann_file.
-
-        Args:
-            ann_file (str): Path of the annotation file.
-
-        Returns:
-            list[dict]: List of annotations sorted by timestamps.
-        """
-        data = mmcv.load(ann_file, file_format='pkl')
-        data_infos = list(sorted(data['infos'], key=lambda e: e['timestamp']))
-        data_infos = data_infos[::self.load_interval]
-        self.metadata = data['metadata']
-        self.version = self.metadata['version']
-        return data_infos
-
-    def get_data_info(self, index):
-        """Get data info according to the given index.
-
-        Args:
-            index (int): Index of the sample data to get.
-
-        Returns:
-            dict: Data information that will be passed to the data
-                preprocessing pipelines. It includes the following keys:
-
-                - sample_idx (str): Sample index.
-                - pts_filename (str): Filename of point clouds.
-                - sweeps (list[dict]): Infos of sweeps.
-                - timestamp (float): Sample timestamp.
-                - img_filename (str, optional): Image filename.
-                - lidar2img (list[np.ndarray], optional): Transformations
-                    from lidar to different cameras.
-                - ann_info (dict): Annotation info.
-        """
-        info = self.data_infos[index]
-        # standard protocol modified from SECOND.Pytorch
-        input_dict = dict(
-            sample_idx=info['token'],
-            pts_filename=info['lidar_path'],
-            sweeps=info['sweeps'],
-            timestamp=info['timestamp'] / 1e6,
-        )
-
-        if self.modality['use_camera']:
-            image_paths = []
-            lidar2img_rts = []
-            for cam_type, cam_info in info['cams'].items():
-                image_paths.append(cam_info['data_path'])
-                # obtain lidar to image transformation matrix
-                lidar2cam_r = np.linalg.inv(cam_info['sensor2lidar_rotation'])
-                lidar2cam_t = cam_info[
-                    'sensor2lidar_translation'] @ lidar2cam_r.T
-                lidar2cam_rt = np.eye(4)
-                lidar2cam_rt[:3, :3] = lidar2cam_r.T
-                lidar2cam_rt[3, :3] = -lidar2cam_t
-                intrinsic = cam_info['cam_intrinsic']
-                viewpad = np.eye(4)
-                viewpad[:intrinsic.shape[0], :intrinsic.shape[1]] = intrinsic
-                lidar2img_rt = (viewpad @ lidar2cam_rt.T)
-                lidar2img_rts.append(lidar2img_rt)
-
-            input_dict.update(
-                dict(
-                    img_filename=image_paths,
-                    lidar2img=lidar2img_rts,
-                ))
-
-        if not self.test_mode:
-            annos = self.get_ann_info(index)
-            input_dict['ann_info'] = annos
-
-        return input_dict
-
-    def get_ann_info(self, index):
+    def parse_ann_info(self, info: dict) -> dict:
         """Get annotation info according to the given index.
 
         Args:
-            index (int): Index of the annotation data to get.
+            info (dict): Data information of single data sample.
 
         Returns:
-            dict: Annotation information consists of the following keys:
+            dict: annotation information consists of the following keys:
 
                 - gt_bboxes_3d (:obj:`LiDARInstance3DBoxes`):
-                    3D ground truth bboxes
+                    3D ground truth bboxes.
                 - gt_labels_3d (np.ndarray): Labels of ground truths.
-                - gt_names (list[str]): Class names of ground truths.
         """
-        info = self.data_infos[index]
-        # filter out bbox containing no points
+        ann_info = super().parse_ann_info(info)
+        if ann_info is None:
+            # empty instance
+            anns_results = dict()
+            anns_results['gt_bboxes_3d'] = np.zeros((0, 7), dtype=np.float32)
+            anns_results['gt_labels_3d'] = np.zeros(0, dtype=np.int64)
+            return anns_results
         if self.use_valid_flag:
-            mask = info['valid_flag']
+            mask = ann_info['bbox_3d_isvalid']
         else:
-            mask = info['num_lidar_pts'] > 0
-        gt_bboxes_3d = info['gt_boxes'][mask]
-        gt_names_3d = info['gt_names'][mask]
-        gt_labels_3d = []
-        for cat in gt_names_3d:
-            if cat in self.CLASSES:
-                gt_labels_3d.append(self.CLASSES.index(cat))
-            else:
-                gt_labels_3d.append(-1)
-        gt_labels_3d = np.array(gt_labels_3d)
+            mask = ann_info['num_lidar_pts'] > 0
+        gt_bboxes_3d = ann_info['gt_bboxes_3d'][mask]
+        gt_labels_3d = ann_info['gt_labels_3d'][mask]
 
         if self.with_velocity:
-            gt_velocity = info['gt_velocity'][mask]
+            gt_velocity = ann_info['velocity'][mask]
             nan_mask = np.isnan(gt_velocity[:, 0])
             gt_velocity[nan_mask] = [0.0, 0.0]
             gt_bboxes_3d = np.concatenate([gt_bboxes_3d, gt_velocity], axis=-1)
@@ -293,362 +118,5 @@ class NuScenesDataset(Det3DDataset):
             origin=(0.5, 0.5, 0.5)).convert_to(self.box_mode_3d)
 
         anns_results = dict(
-            gt_bboxes_3d=gt_bboxes_3d,
-            gt_labels_3d=gt_labels_3d,
-            gt_names=gt_names_3d)
+            gt_bboxes_3d=gt_bboxes_3d, gt_labels_3d=gt_labels_3d)
         return anns_results
-
-    def _format_bbox(self, results, jsonfile_prefix=None):
-        """Convert the results to the standard format.
-
-        Args:
-            results (list[dict]): Testing results of the dataset.
-            jsonfile_prefix (str): The prefix of the output jsonfile.
-                You can specify the output directory/filename by
-                modifying the jsonfile_prefix. Default: None.
-
-        Returns:
-            str: Path of the output json file.
-        """
-        nusc_annos = {}
-        mapped_class_names = self.CLASSES
-
-        print('Start to convert detection format...')
-        for sample_id, det in enumerate(mmcv.track_iter_progress(results)):
-            annos = []
-            boxes = output_to_nusc_box(det)
-            sample_token = self.data_infos[sample_id]['token']
-            boxes = lidar_nusc_box_to_global(self.data_infos[sample_id], boxes,
-                                             mapped_class_names,
-                                             self.eval_detection_configs,
-                                             self.eval_version)
-            for i, box in enumerate(boxes):
-                name = mapped_class_names[box.label]
-                if np.sqrt(box.velocity[0]**2 + box.velocity[1]**2) > 0.2:
-                    if name in [
-                            'car',
-                            'construction_vehicle',
-                            'bus',
-                            'truck',
-                            'trailer',
-                    ]:
-                        attr = 'vehicle.moving'
-                    elif name in ['bicycle', 'motorcycle']:
-                        attr = 'cycle.with_rider'
-                    else:
-                        attr = NuScenesDataset.DefaultAttribute[name]
-                else:
-                    if name in ['pedestrian']:
-                        attr = 'pedestrian.standing'
-                    elif name in ['bus']:
-                        attr = 'vehicle.stopped'
-                    else:
-                        attr = NuScenesDataset.DefaultAttribute[name]
-
-                nusc_anno = dict(
-                    sample_token=sample_token,
-                    translation=box.center.tolist(),
-                    size=box.wlh.tolist(),
-                    rotation=box.orientation.elements.tolist(),
-                    velocity=box.velocity[:2].tolist(),
-                    detection_name=name,
-                    detection_score=box.score,
-                    attribute_name=attr)
-                annos.append(nusc_anno)
-            nusc_annos[sample_token] = annos
-        nusc_submissions = {
-            'meta': self.modality,
-            'results': nusc_annos,
-        }
-
-        mmcv.mkdir_or_exist(jsonfile_prefix)
-        res_path = osp.join(jsonfile_prefix, 'results_nusc.json')
-        print('Results writes to', res_path)
-        mmcv.dump(nusc_submissions, res_path)
-        return res_path
-
-    def _evaluate_single(self,
-                         result_path,
-                         logger=None,
-                         metric='bbox',
-                         result_name='pts_bbox'):
-        """Evaluation for a single model in nuScenes protocol.
-
-        Args:
-            result_path (str): Path of the result file.
-            logger (logging.Logger | str, optional): Logger used for printing
-                related information during evaluation. Default: None.
-            metric (str, optional): Metric name used for evaluation.
-                Default: 'bbox'.
-            result_name (str, optional): Result name in the metric prefix.
-                Default: 'pts_bbox'.
-
-        Returns:
-            dict: Dictionary of evaluation details.
-        """
-        from nuscenes import NuScenes
-        from nuscenes.eval.detection.evaluate import NuScenesEval
-
-        output_dir = osp.join(*osp.split(result_path)[:-1])
-        nusc = NuScenes(
-            version=self.version, dataroot=self.data_root, verbose=False)
-        eval_set_map = {
-            'v1.0-mini': 'mini_val',
-            'v1.0-trainval': 'val',
-        }
-        nusc_eval = NuScenesEval(
-            nusc,
-            config=self.eval_detection_configs,
-            result_path=result_path,
-            eval_set=eval_set_map[self.version],
-            output_dir=output_dir,
-            verbose=False)
-        nusc_eval.main(render_curves=False)
-
-        # record metrics
-        metrics = mmcv.load(osp.join(output_dir, 'metrics_summary.json'))
-        detail = dict()
-        metric_prefix = f'{result_name}_NuScenes'
-        for name in self.CLASSES:
-            for k, v in metrics['label_aps'][name].items():
-                val = float('{:.4f}'.format(v))
-                detail['{}/{}_AP_dist_{}'.format(metric_prefix, name, k)] = val
-            for k, v in metrics['label_tp_errors'][name].items():
-                val = float('{:.4f}'.format(v))
-                detail['{}/{}_{}'.format(metric_prefix, name, k)] = val
-            for k, v in metrics['tp_errors'].items():
-                val = float('{:.4f}'.format(v))
-                detail['{}/{}'.format(metric_prefix,
-                                      self.ErrNameMapping[k])] = val
-
-        detail['{}/NDS'.format(metric_prefix)] = metrics['nd_score']
-        detail['{}/mAP'.format(metric_prefix)] = metrics['mean_ap']
-        return detail
-
-    def format_results(self, results, jsonfile_prefix=None):
-        """Format the results to json (standard format for COCO evaluation).
-
-        Args:
-            results (list[dict]): Testing results of the dataset.
-            jsonfile_prefix (str): The prefix of json files. It includes
-                the file path and the prefix of filename, e.g., "a/b/prefix".
-                If not specified, a temp file will be created. Default: None.
-
-        Returns:
-            tuple: Returns (result_files, tmp_dir), where `result_files` is a
-                dict containing the json filepaths, `tmp_dir` is the temporal
-                directory created for saving json files when
-                `jsonfile_prefix` is not specified.
-        """
-        assert isinstance(results, list), 'results must be a list'
-        assert len(results) == len(self), (
-            'The length of results is not equal to the dataset len: {} != {}'.
-            format(len(results), len(self)))
-
-        if jsonfile_prefix is None:
-            tmp_dir = tempfile.TemporaryDirectory()
-            jsonfile_prefix = osp.join(tmp_dir.name, 'results')
-        else:
-            tmp_dir = None
-
-        # currently the output prediction results could be in two formats
-        # 1. list of dict('boxes_3d': ..., 'scores_3d': ..., 'labels_3d': ...)
-        # 2. list of dict('pts_bbox' or 'img_bbox':
-        #     dict('boxes_3d': ..., 'scores_3d': ..., 'labels_3d': ...))
-        # this is a workaround to enable evaluation of both formats on nuScenes
-        # refer to https://github.com/open-mmlab/mmdetection3d/issues/449
-        if not ('pts_bbox' in results[0] or 'img_bbox' in results[0]):
-            result_files = self._format_bbox(results, jsonfile_prefix)
-        else:
-            # should take the inner dict out of 'pts_bbox' or 'img_bbox' dict
-            result_files = dict()
-            for name in results[0]:
-                print(f'\nFormating bboxes of {name}')
-                results_ = [out[name] for out in results]
-                tmp_file_ = osp.join(jsonfile_prefix, name)
-                result_files.update(
-                    {name: self._format_bbox(results_, tmp_file_)})
-        return result_files, tmp_dir
-
-    def evaluate(self,
-                 results,
-                 metric='bbox',
-                 logger=None,
-                 jsonfile_prefix=None,
-                 result_names=['pts_bbox'],
-                 show=False,
-                 out_dir=None,
-                 pipeline=None):
-        """Evaluation in nuScenes protocol.
-
-        Args:
-            results (list[dict]): Testing results of the dataset.
-            metric (str | list[str], optional): Metrics to be evaluated.
-                Default: 'bbox'.
-            logger (logging.Logger | str, optional): Logger used for printing
-                related information during evaluation. Default: None.
-            jsonfile_prefix (str, optional): The prefix of json files including
-                the file path and the prefix of filename, e.g., "a/b/prefix".
-                If not specified, a temp file will be created. Default: None.
-            show (bool, optional): Whether to visualize.
-                Default: False.
-            out_dir (str, optional): Path to save the visualization results.
-                Default: None.
-            pipeline (list[dict], optional): raw data loading for showing.
-                Default: None.
-
-        Returns:
-            dict[str, float]: Results of each evaluation metric.
-        """
-        result_files, tmp_dir = self.format_results(results, jsonfile_prefix)
-
-        if isinstance(result_files, dict):
-            results_dict = dict()
-            for name in result_names:
-                print('Evaluating bboxes of {}'.format(name))
-                ret_dict = self._evaluate_single(result_files[name])
-            results_dict.update(ret_dict)
-        elif isinstance(result_files, str):
-            results_dict = self._evaluate_single(result_files)
-
-        if tmp_dir is not None:
-            tmp_dir.cleanup()
-
-        if show or out_dir:
-            self.show(results, out_dir, show=show, pipeline=pipeline)
-        return results_dict
-
-    def _build_default_pipeline(self):
-        """Build the default pipeline for this dataset."""
-        pipeline = [
-            dict(
-                type='LoadPointsFromFile',
-                coord_type='LIDAR',
-                load_dim=5,
-                use_dim=5,
-                file_client_args=dict(backend='disk')),
-            dict(
-                type='LoadPointsFromMultiSweeps',
-                sweeps_num=10,
-                file_client_args=dict(backend='disk')),
-            dict(
-                type='DefaultFormatBundle3D',
-                class_names=self.CLASSES,
-                with_label=False),
-            dict(type='Collect3D', keys=['points'])
-        ]
-        return Compose(pipeline)
-
-    def show(self, results, out_dir, show=False, pipeline=None):
-        """Results visualization.
-
-        Args:
-            results (list[dict]): List of bounding boxes results.
-            out_dir (str): Output directory of visualization result.
-            show (bool): Whether to visualize the results online.
-                Default: False.
-            pipeline (list[dict], optional): raw data loading for showing.
-                Default: None.
-        """
-        assert out_dir is not None, 'Expect out_dir, got none.'
-        pipeline = self._get_pipeline(pipeline)
-        for i, result in enumerate(results):
-            if 'pts_bbox' in result.keys():
-                result = result['pts_bbox']
-            data_info = self.data_infos[i]
-            pts_path = data_info['lidar_path']
-            file_name = osp.split(pts_path)[-1].split('.')[0]
-            points = self._extract_data(i, pipeline, 'points').numpy()
-            # for now we convert points into depth mode
-            points = Coord3DMode.convert_point(points, Coord3DMode.LIDAR,
-                                               Coord3DMode.DEPTH)
-            inds = result['scores_3d'] > 0.1
-            gt_bboxes = self.get_ann_info(i)['gt_bboxes_3d'].tensor.numpy()
-            show_gt_bboxes = Box3DMode.convert(gt_bboxes, Box3DMode.LIDAR,
-                                               Box3DMode.DEPTH)
-            pred_bboxes = result['boxes_3d'][inds].tensor.numpy()
-            show_pred_bboxes = Box3DMode.convert(pred_bboxes, Box3DMode.LIDAR,
-                                                 Box3DMode.DEPTH)
-            show_result(points, show_gt_bboxes, show_pred_bboxes, out_dir,
-                        file_name, show)
-
-
-def output_to_nusc_box(detection):
-    """Convert the output to the box class in the nuScenes.
-
-    Args:
-        detection (dict): Detection results.
-
-            - boxes_3d (:obj:`BaseInstance3DBoxes`): Detection bbox.
-            - scores_3d (torch.Tensor): Detection scores.
-            - labels_3d (torch.Tensor): Predicted box labels.
-
-    Returns:
-        list[:obj:`NuScenesBox`]: List of standard NuScenesBoxes.
-    """
-    box3d = detection['boxes_3d']
-    scores = detection['scores_3d'].numpy()
-    labels = detection['labels_3d'].numpy()
-
-    box_gravity_center = box3d.gravity_center.numpy()
-    box_dims = box3d.dims.numpy()
-    box_yaw = box3d.yaw.numpy()
-
-    # our LiDAR coordinate system -> nuScenes box coordinate system
-    nus_box_dims = box_dims[:, [1, 0, 2]]
-
-    box_list = []
-    for i in range(len(box3d)):
-        quat = pyquaternion.Quaternion(axis=[0, 0, 1], radians=box_yaw[i])
-        velocity = (*box3d.tensor[i, 7:9], 0.0)
-        # velo_val = np.linalg.norm(box3d[i, 7:9])
-        # velo_ori = box3d[i, 6]
-        # velocity = (
-        # velo_val * np.cos(velo_ori), velo_val * np.sin(velo_ori), 0.0)
-        box = NuScenesBox(
-            box_gravity_center[i],
-            nus_box_dims[i],
-            quat,
-            label=labels[i],
-            score=scores[i],
-            velocity=velocity)
-        box_list.append(box)
-    return box_list
-
-
-def lidar_nusc_box_to_global(info,
-                             boxes,
-                             classes,
-                             eval_configs,
-                             eval_version='detection_cvpr_2019'):
-    """Convert the box from ego to global coordinate.
-
-    Args:
-        info (dict): Info for a specific sample data, including the
-            calibration information.
-        boxes (list[:obj:`NuScenesBox`]): List of predicted NuScenesBoxes.
-        classes (list[str]): Mapped classes in the evaluation.
-        eval_configs (object): Evaluation configuration object.
-        eval_version (str, optional): Evaluation version.
-            Default: 'detection_cvpr_2019'
-
-    Returns:
-        list: List of standard NuScenesBoxes in the global
-            coordinate.
-    """
-    box_list = []
-    for box in boxes:
-        # Move box to ego vehicle coord system
-        box.rotate(pyquaternion.Quaternion(info['lidar2ego_rotation']))
-        box.translate(np.array(info['lidar2ego_translation']))
-        # filter det in ego.
-        cls_range_map = eval_configs.class_range
-        radius = np.linalg.norm(box.center[:2], 2)
-        det_range = cls_range_map[classes[box.label]]
-        if radius > det_range:
-            continue
-        # Move box to global coord system
-        box.rotate(pyquaternion.Quaternion(info['ego2global_rotation']))
-        box.translate(np.array(info['ego2global_translation']))
-        box_list.append(box)
-    return box_list

mmdet3d/datasets/pipelines/loading.py

@@ -101,7 +101,7 @@ class LoadImageFromFileMono3D(LoadImageFromFile):
 
 
 @TRANSFORMS.register_module()
-class LoadPointsFromMultiSweeps(object):
+class LoadPointsFromMultiSweeps(BaseTransform):
     """Load points from multiple sweeps.
 
     This is usually used for nuScenes dataset to utilize previous sweeps.
@@ -186,7 +186,7 @@ class LoadPointsFromMultiSweeps(object):
         not_close = np.logical_not(np.logical_and(x_filt, y_filt))
         return points[not_close]
 
-    def __call__(self, results):
+    def transform(self, results):
         """Call function to load multi-sweep point clouds from files.
 
         Args:
@@ -204,30 +204,35 @@ class LoadPointsFromMultiSweeps(object):
         points.tensor[:, 4] = 0
         sweep_points_list = [points]
         ts = results['timestamp']
-        if self.pad_empty_sweeps and len(results['sweeps']) == 0:
-            for i in range(self.sweeps_num):
-                if self.remove_close:
-                    sweep_points_list.append(self._remove_close(points))
-                else:
-                    sweep_points_list.append(points)
+        if 'lidar_sweeps' not in results:
+            if self.pad_empty_sweeps:
+                for i in range(self.sweeps_num):
+                    if self.remove_close:
+                        sweep_points_list.append(self._remove_close(points))
+                    else:
+                        sweep_points_list.append(points)
         else:
-            if len(results['sweeps']) <= self.sweeps_num:
-                choices = np.arange(len(results['sweeps']))
+            if len(results['lidar_sweeps']) <= self.sweeps_num:
+                choices = np.arange(len(results['lidar_sweeps']))
             elif self.test_mode:
                 choices = np.arange(self.sweeps_num)
             else:
                 choices = np.random.choice(
-                    len(results['sweeps']), self.sweeps_num, replace=False)
+                    len(results['lidar_sweeps']),
+                    self.sweeps_num,
+                    replace=False)
             for idx in choices:
-                sweep = results['sweeps'][idx]
-                points_sweep = self._load_points(sweep['data_path'])
+                sweep = results['lidar_sweeps'][idx]
+                points_sweep = self._load_points(
+                    sweep['lidar_points']['lidar_path'])
                 points_sweep = np.copy(points_sweep).reshape(-1, self.load_dim)
                 if self.remove_close:
                     points_sweep = self._remove_close(points_sweep)
-                sweep_ts = sweep['timestamp'] / 1e6
-                points_sweep[:, :3] = points_sweep[:, :3] @ sweep[
-                    'sensor2lidar_rotation'].T
-                points_sweep[:, :3] += sweep['sensor2lidar_translation']
+                # bc-breaking: Timestamp has divided 1e6 in pkl infos.
+                sweep_ts = sweep['timestamp']
+                lidar2cam = np.array(sweep['lidar_points']['lidar2sensor'])
+                points_sweep[:, :3] = points_sweep[:, :3] @ lidar2cam[:3, :3]
+                points_sweep[:, :3] -= lidar2cam[:3, 3]
                 points_sweep[:, 4] = ts - sweep_ts
                 points_sweep = points.new_point(points_sweep)
                 sweep_points_list.append(points_sweep)

mmdet3d/datasets/utils.py

 # Copyright (c) OpenMMLab. All rights reserved.
 import mmcv
+import numpy as np
 from mmcv.transforms import LoadImageFromFile
+from pyquaternion import Quaternion
 
 # yapf: disable
 from mmdet3d.datasets.pipelines import (LoadAnnotations3D,
@@ -137,3 +139,14 @@ def extract_result_dict(results, key):
     if isinstance(data, mmcv.parallel.DataContainer):
         data = data._data
     return data
+
+
+def convert_quaternion_to_matrix(quaternion: list,
+                                 translation: list = None) -> list:
+    """Compute a transform matrix by given quaternion and translation
+    vector."""
+    result = np.eye(4)
+    result[:3, :3] = Quaternion(quaternion).rotation_matrix
+    if translation is not None:
+        result[:3, 3] = np.array(translation)
+    return result.astype(np.float32).tolist()

tests/test_data/test_datasets/test_nuscenes_dataset.py

+# Copyright (c) OpenMMLab. All rights reserved.
+import numpy as np
+from mmcv.transforms.base import BaseTransform
+from mmengine.data import InstanceData
+from mmengine.registry import TRANSFORMS
+
+from mmdet3d.core.bbox import LiDARInstance3DBoxes
+from mmdet3d.core.data_structures import Det3DDataSample
+from mmdet3d.datasets import NuScenesDataset
+
+
+def _generate_nus_dataset_config():
+    data_root = 'tests/data/nuscenes'
+    ann_file = 'nus_info.pkl'
+    classes = [
+        'car', 'truck', 'trailer', 'bus', 'construction_vehicle', 'bicycle',
+        'motorcycle', 'pedestrian', 'traffic_cone', 'barrier'
+    ]
+    if 'Identity' not in TRANSFORMS:
+
+        @TRANSFORMS.register_module()
+        class Identity(BaseTransform):
+
+            def transform(self, info):
+                packed_input = dict(data_sample=Det3DDataSample())
+                if 'ann_info' in info:
+                    packed_input['data_sample'].gt_instances_3d = InstanceData(
+                    )
+                    packed_input[
+                        'data_sample'].gt_instances_3d.labels_3d = info[
+                            'ann_info']['gt_labels_3d']
+                return packed_input
+
+    pipeline = [
+        dict(type='Identity'),
+    ]
+    modality = dict(use_lidar=True, use_camera=True)
+    data_prefix = dict(pts='samples/LIDAR_TOP', img='samples/CAM_BACK_LEFT')
+    return data_root, ann_file, classes, data_prefix, pipeline, modality
+
+
+def test_getitem():
+    np.random.seed(0)
+    data_root, ann_file, classes, data_prefix, pipeline, modality = \
+        _generate_nus_dataset_config()
+
+    nus_dataset = NuScenesDataset(
+        data_root,
+        ann_file,
+        data_prefix=data_prefix,
+        pipeline=pipeline,
+        metainfo=dict(CLASSES=classes),
+        modality=modality)
+
+    nus_dataset.prepare_data(0)
+    input_dict = nus_dataset.get_data_info(0)
+    # assert the the path should contains data_prefix and data_root
+    assert data_prefix['pts'] in input_dict['lidar_points']['lidar_path']
+    assert input_dict['lidar_points'][
+        'lidar_path'] == 'tests/data/nuscenes/samples/LIDAR_TOP/' \
+                         'n015-2018-08-02-17-16-37+0800__LIDAR_TOP__' \
+                         '1533201470948018.pcd.bin'
+    for cam_id, img_info in input_dict['images'].items():
+        if 'img_path' in img_info:
+            assert data_prefix['img'] in img_info['img_path']
+            assert data_root in img_info['img_path']
+
+    ann_info = nus_dataset.parse_ann_info(input_dict)
+
+    # assert the keys in ann_info and the type
+    assert 'gt_labels_3d' in ann_info
+    assert ann_info['gt_labels_3d'].dtype == np.int64
+    assert len(ann_info['gt_labels_3d']) == 37
+
+    assert 'gt_bboxes_3d' in ann_info
+    assert isinstance(ann_info['gt_bboxes_3d'], LiDARInstance3DBoxes)
+
+    assert len(nus_dataset.metainfo['CLASSES']) == 10
+
+    assert input_dict['token'] == 'fd8420396768425eabec9bdddf7e64b6'
+    assert input_dict['timestamp'] == 1533201470.448696

tools/data_converter/update_infos_to_v2.py

@@ -15,6 +15,8 @@ from os import path as osp
 import mmcv
 import numpy as np
 
+from mmdet3d.datasets.utils import convert_quaternion_to_matrix
+
 
 def get_empty_instance():
     """Empty annotation for single instance."""
@@ -156,6 +158,7 @@ def get_empty_standard_data_info():
         radar_points=get_empty_radar_points(),
         # (list[dict], optional): Image sweeps data.
         image_sweeps=[],
+        lidar_sweeps=[],
         instances=[],
         # (list[dict], optional): Required by object
         # detection, instance  to be ignored during training.
@@ -203,6 +206,116 @@ def clear_data_info_unused_keys(data_info):
     return data_info, empty_flag
 
 
+def update_nuscenes_infos(pkl_path, out_dir):
+    print(f'{pkl_path} will be modified.')
+    if out_dir in pkl_path:
+        print(f'Warning, you may overwriting '
+              f'the original data {pkl_path}.')
+    print(f'Reading from input file: {pkl_path}.')
+    data_list = mmcv.load(pkl_path)
+    METAINFO = {
+        'CLASSES':
+        ('car', 'truck', 'trailer', 'bus', 'construction_vehicle', 'bicycle',
+         'motorcycle', 'pedestrian', 'traffic_cone', 'barrier'),
+        'DATASET':
+        'Nuscenes',
+        'version':
+        data_list['metadata']['version']
+    }
+    print('Start updating:')
+    converted_list = []
+    for i, ori_info_dict in enumerate(
+            mmcv.track_iter_progress(data_list['infos'])):
+        temp_data_info = get_empty_standard_data_info()
+        temp_data_info['sample_idx'] = i
+        temp_data_info['token'] = ori_info_dict['token']
+        temp_data_info['ego2global'] = convert_quaternion_to_matrix(
+            ori_info_dict['ego2global_rotation'],
+            ori_info_dict['ego2global_translation'])
+        temp_data_info['lidar_points']['lidar_path'] = ori_info_dict[
+            'lidar_path'].split('/')[-1]
+        temp_data_info['lidar_points'][
+            'lidar2ego'] = convert_quaternion_to_matrix(
+                ori_info_dict['lidar2ego_rotation'],
+                ori_info_dict['lidar2ego_translation'])
+        # bc-breaking: Timestamp has divided 1e6 in pkl infos.
+        temp_data_info['timestamp'] = ori_info_dict['timestamp'] / 1e6
+        for ori_sweep in ori_info_dict['sweeps']:
+            temp_lidar_sweep = get_single_lidar_sweep()
+            temp_lidar_sweep['lidar_points'][
+                'lidar2ego'] = convert_quaternion_to_matrix(
+                    ori_sweep['sensor2ego_rotation'],
+                    ori_sweep['sensor2ego_translation'])
+            temp_lidar_sweep['ego2global'] = convert_quaternion_to_matrix(
+                ori_sweep['ego2global_rotation'],
+                ori_sweep['ego2global_translation'])
+            lidar2sensor = np.eye(4)
+            lidar2sensor[:3, :3] = ori_sweep['sensor2lidar_rotation'].T
+            lidar2sensor[:3, 3] = -ori_sweep['sensor2lidar_translation']
+            temp_lidar_sweep['lidar_points'][
+                'lidar2sensor'] = lidar2sensor.astype(np.float32).tolist()
+            temp_lidar_sweep['timestamp'] = ori_sweep['timestamp'] / 1e6
+            temp_lidar_sweep['lidar_points']['lidar_path'] = ori_sweep[
+                'data_path']
+            temp_lidar_sweep['sample_data_token'] = ori_sweep[
+                'sample_data_token']
+            temp_data_info['lidar_sweeps'].append(temp_lidar_sweep)
+        temp_data_info['images'] = {}
+        for cam in ori_info_dict['cams']:
+            empty_img_info = get_empty_img_info()
+            empty_img_info['img_path'] = ori_info_dict['cams'][cam][
+                'data_path'].split('/')[-1]
+            empty_img_info['cam2img'] = ori_info_dict['cams'][cam][
+                'cam_intrinsic'].tolist()
+            empty_img_info['sample_data_token'] = ori_info_dict['cams'][cam][
+                'sample_data_token']
+            # bc-breaking: Timestamp has divided 1e6 in pkl infos.
+            empty_img_info[
+                'timestamp'] = ori_info_dict['cams'][cam]['timestamp'] / 1e6
+            empty_img_info['cam2ego'] = convert_quaternion_to_matrix(
+                ori_info_dict['cams'][cam]['sensor2ego_rotation'],
+                ori_info_dict['cams'][cam]['sensor2ego_translation'])
+            lidar2sensor = np.eye(4)
+            lidar2sensor[:3, :3] = ori_info_dict['cams'][cam][
+                'sensor2lidar_rotation'].T
+            lidar2sensor[:3, 3] = -ori_info_dict['cams'][cam][
+                'sensor2lidar_translation']
+            empty_img_info['lidar2cam'] = lidar2sensor.astype(
+                np.float32).tolist()
+            temp_data_info['images'][cam] = empty_img_info
+        num_instances = ori_info_dict['gt_boxes'].shape[0]
+        ignore_class_name = set()
+        for i in range(num_instances):
+            empty_instance = get_empty_instance()
+            empty_instance['bbox_3d'] = ori_info_dict['gt_boxes'][
+                i, :].tolist()
+            if ori_info_dict['gt_names'][i] in METAINFO['CLASSES']:
+                empty_instance['bbox_label'] = METAINFO['CLASSES'].index(
+                    ori_info_dict['gt_names'][i])
+            else:
+                ignore_class_name.add(ori_info_dict['gt_names'][i])
+                empty_instance['bbox_label'] = -1
+            empty_instance['bbox_label_3d'] = copy.deepcopy(
+                empty_instance['bbox_label'])
+            empty_instance['velocity'] = ori_info_dict['gt_velocity'][
+                i, :].tolist()
+            empty_instance['num_lidar_pts'] = ori_info_dict['num_lidar_pts'][i]
+            empty_instance['num_radar_pts'] = ori_info_dict['num_radar_pts'][i]
+            empty_instance['bbox_3d_isvalid'] = ori_info_dict['valid_flag'][i]
+            empty_instance = clear_instance_unused_keys(empty_instance)
+            temp_data_info['instances'].append(empty_instance)
+        temp_data_info, _ = clear_data_info_unused_keys(temp_data_info)
+        converted_list.append(temp_data_info)
+    pkl_name = pkl_path.split('/')[-1]
+    out_path = osp.join(out_dir, pkl_name)
+    print(f'Writing to output file: {out_path}.')
+    print(f'ignore classes: {ignore_class_name}')
+    converted_data_info = dict(metainfo=METAINFO, data_list=converted_list)
+
+    mmcv.dump(converted_data_info, out_path, 'pkl')
+    return temp_lidar_sweep
+
+
 def update_kitti_infos(pkl_path, out_dir):
     print(f'{pkl_path} will be modified.')
     if out_dir in pkl_path:
@@ -479,6 +592,8 @@ def main():
         update_scannet_infos(pkl_path=args.pkl, out_dir=args.out_dir)
     elif args.dataset.lower() == 'sunrgbd':
         update_sunrgbd_infos(pkl_path=args.pkl, out_dir=args.out_dir)
+    elif args.dataset.lower() == 'nuscenes':
+        update_nuscenes_infos(pkl_path=args.pkl, out_dir=args.out_dir)
     else:
         raise NotImplementedError(
             f'Do not support convert {args.dataset} to v2.')