Release v1.1.0rc1

Release v1.1.0rc1

tests/test_engine/test_hooks/test_visualization_hook.py

@@ -5,42 +5,59 @@ import time
 from unittest import TestCase
 from unittest.mock import Mock
 
+import numpy as np
 import torch
 from mmengine.structures import InstanceData
 
 from mmdet3d.engine.hooks import Det3DVisualizationHook
-from mmdet3d.structures import Det3DDataSample
+from mmdet3d.structures import Det3DDataSample, LiDARInstance3DBoxes
 from mmdet3d.visualization import Det3DLocalVisualizer
 
 
-def _rand_bboxes(num_boxes, h, w):
-    cx, cy, bw, bh = torch.rand(num_boxes, 4).T
-
-    tl_x = ((cx * w) - (w * bw / 2)).clip(0, w)
-    tl_y = ((cy * h) - (h * bh / 2)).clip(0, h)
-    br_x = ((cx * w) + (w * bw / 2)).clip(0, w)
-    br_y = ((cy * h) + (h * bh / 2)).clip(0, h)
-
-    bboxes = torch.vstack([tl_x, tl_y, br_x, br_y]).T
-    return bboxes
-
-
 class TestVisualizationHook(TestCase):
 
     def setUp(self) -> None:
         Det3DLocalVisualizer.get_instance('visualizer')
 
-        pred_instances = InstanceData()
-        pred_instances.bboxes = _rand_bboxes(5, 10, 12)
-        pred_instances.labels = torch.randint(0, 2, (5, ))
-        pred_instances.scores = torch.rand((5, ))
-        pred_det_data_sample = Det3DDataSample()
-        pred_det_data_sample.set_metainfo({
+        pred_instances_3d = InstanceData()
+        pred_instances_3d.bboxes_3d = LiDARInstance3DBoxes(
+            torch.tensor(
+                [[8.7314, -1.8559, -1.5997, 1.2000, 0.4800, 1.8900, -1.5808]]))
+        pred_instances_3d.labels_3d = torch.tensor([0])
+        pred_instances_3d.scores_3d = torch.tensor([0.8])
+
+        pred_det3d_data_sample = Det3DDataSample()
+        pred_det3d_data_sample.set_metainfo({
+            'num_pts_feats':
+            4,
+            'lidar2img':
+            np.array([[
+                6.02943734e+02, -7.07913286e+02, -1.22748427e+01,
+                -1.70942724e+02
+            ],
+                      [
+                          1.76777261e+02, 8.80879902e+00, -7.07936120e+02,
+                          -1.02568636e+02
+                      ],
+                      [
+                          9.99984860e-01, -1.52826717e-03, -5.29071223e-03,
+                          -3.27567990e-01
+                      ],
+                      [
+                          0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
+                          1.00000000e+00
+                      ]]),
             'img_path':
-            osp.join(osp.dirname(__file__), '../../data/color.jpg')
+            osp.join(
+                osp.dirname(__file__),
+                '../../data/kitti/training/image_2/000000.png'),
+            'lidar_path':
+            osp.join(
+                osp.dirname(__file__),
+                '../../data/kitti/training/velodyne_reduced/000000.bin')
         })
-        pred_det_data_sample.pred_instances = pred_instances
-        self.outputs = [pred_det_data_sample] * 2
+        pred_det3d_data_sample.pred_instances_3d = pred_instances_3d
+        self.outputs = [pred_det3d_data_sample] * 2
 
     def test_after_val_iter(self):
         runner = Mock()

tests/test_models/test_dense_heads/test_fcos_mono3d_head.py

@@ -44,9 +44,9 @@ class TestFCOSMono3DHead(TestCase):
 
         fcos_mono3d_head = FCOSMono3DHead(
             num_classes=10,
-            in_channels=256,
+            in_channels=32,
             stacked_convs=2,
-            feat_channels=256,
+            feat_channels=32,
             use_direction_classifier=True,
             diff_rad_by_sin=True,
             pred_attrs=True,
@@ -55,16 +55,16 @@ class TestFCOSMono3DHead(TestCase):
             dir_limit_offset=0,
             strides=[8, 16, 32, 64, 128],
             group_reg_dims=(2, 1, 3, 1, 2),  # offset, depth, size, rot, velo
-            cls_branch=(256, ),
+            cls_branch=(32, ),
             reg_branch=(
-                (256, ),  # offset
-                (256, ),  # depth
-                (256, ),  # size
-                (256, ),  # rot
+                (32, ),  # offset
+                (32, ),  # depth
+                (32, ),  # size
+                (32, ),  # rot
                 ()  # velo
             ),
-            dir_branch=(256, ),
-            attr_branch=(256, ),
+            dir_branch=(32, ),
+            attr_branch=(32, ),
             loss_cls=dict(
                 type='mmdet.FocalLoss',
                 use_sigmoid=True,
@@ -96,11 +96,11 @@ class TestFCOSMono3DHead(TestCase):
 
         # FCOS3D head expects a multiple levels of features per image
         feats = [
-            torch.rand([1, 256, 116, 200], dtype=torch.float32),
-            torch.rand([1, 256, 58, 100], dtype=torch.float32),
-            torch.rand([1, 256, 29, 50], dtype=torch.float32),
-            torch.rand([1, 256, 15, 25], dtype=torch.float32),
-            torch.rand([1, 256, 8, 13], dtype=torch.float32)
+            torch.rand([1, 32, 116, 200], dtype=torch.float32),
+            torch.rand([1, 32, 58, 100], dtype=torch.float32),
+            torch.rand([1, 32, 29, 50], dtype=torch.float32),
+            torch.rand([1, 32, 15, 25], dtype=torch.float32),
+            torch.rand([1, 32, 8, 13], dtype=torch.float32)
         ]
 
         # Test forward

tests/test_models/test_dense_heads/test_freeanchors.py

@@ -17,8 +17,16 @@ class TestFreeAnchor(unittest.TestCase):
         DefaultScope.get_instance('test_freeanchor', scope_name='mmdet3d')
         _setup_seed(0)
         freeanchor_cfg = _get_detector_cfg(
-            'free_anchor/hv_pointpillars_fpn_sbn-all_free-'
-            'anchor_4x8_2x_nus-3d.py')
+            'free_anchor/pointpillars_hv_regnet-1.6gf_fpn_head-free-anchor'
+            '_sbn-all_8xb4-2x_nus-3d.py')
+        # decrease channels to reduce cuda memory.
+        freeanchor_cfg.pts_voxel_encoder.feat_channels = [1, 1]
+        freeanchor_cfg.pts_middle_encoder.in_channels = 1
+        freeanchor_cfg.pts_backbone.base_channels = 1
+        freeanchor_cfg.pts_backbone.stem_channels = 1
+        freeanchor_cfg.pts_neck.out_channels = 1
+        freeanchor_cfg.pts_bbox_head.feat_channels = 1
+        freeanchor_cfg.pts_bbox_head.in_channels = 1
         model = MODELS.build(freeanchor_cfg)
         num_gt_instance = 3
         packed_inputs = _create_detector_inputs(

tests/test_models/test_dense_heads/test_ssn.py

@@ -18,6 +18,14 @@ class TestSSN(unittest.TestCase):
         _setup_seed(0)
         ssn_cfg = _get_detector_cfg(
             'ssn/ssn_hv_secfpn_sbn-all_16xb2-2x_nus-3d.py')
+        ssn_cfg.pts_voxel_encoder.feat_channels = [1, 1]
+        ssn_cfg.pts_middle_encoder.in_channels = 1
+        ssn_cfg.pts_backbone.in_channels = 1
+        ssn_cfg.pts_backbone.out_channels = [1, 1, 1]
+        ssn_cfg.pts_neck.in_channels = [1, 1, 1]
+        ssn_cfg.pts_neck.out_channels = [1, 1, 1]
+        ssn_cfg.pts_bbox_head.in_channels = 3
+        ssn_cfg.pts_bbox_head.feat_channels = 1
         model = MODELS.build(ssn_cfg)
         num_gt_instance = 50
         packed_inputs = _create_detector_inputs(

tests/test_models/test_detectors/test_pointrcnn.py

+import unittest
+
+import torch
+from mmengine import DefaultScope
+
+from mmdet3d.registry import MODELS
+from tests.utils.model_utils import (_create_detector_inputs,
+                                     _get_detector_cfg, _setup_seed)
+
+
+class TestPointRCNN(unittest.TestCase):
+
+    def test_pointrcnn(self):
+        import mmdet3d.models
+
+        assert hasattr(mmdet3d.models, 'PointRCNN')
+        DefaultScope.get_instance('test_pointrcnn', scope_name='mmdet3d')
+        _setup_seed(0)
+        pointrcnn_cfg = _get_detector_cfg(
+            'point_rcnn/point-rcnn_8xb2_kitti-3d-3class.py')
+        model = MODELS.build(pointrcnn_cfg)
+        num_gt_instance = 2
+        packed_inputs = _create_detector_inputs(
+            num_points=10101, num_gt_instance=num_gt_instance)
+
+        if torch.cuda.is_available():
+            model = model.cuda()
+            # test simple_test
+            with torch.no_grad():
+                data = model.data_preprocessor(packed_inputs, True)
+                torch.cuda.empty_cache()
+                results = model.forward(**data, mode='predict')
+            self.assertEqual(len(results), 1)
+            self.assertIn('bboxes_3d', results[0].pred_instances_3d)
+            self.assertIn('scores_3d', results[0].pred_instances_3d)
+            self.assertIn('labels_3d', results[0].pred_instances_3d)
+
+            # save the memory
+            with torch.no_grad():
+                losses = model.forward(**data, mode='loss')
+                torch.cuda.empty_cache()
+            self.assertGreaterEqual(losses['rpn_bbox_loss'], 0)
+            self.assertGreaterEqual(losses['rpn_semantic_loss'], 0)
+            self.assertGreaterEqual(losses['loss_cls'], 0)
+            self.assertGreaterEqual(losses['loss_bbox'], 0)
+            self.assertGreaterEqual(losses['loss_corner'], 0)

tests/test_structures/test_bbox/test_box3d.py

@@ -566,7 +566,7 @@ def test_lidar_boxes3d():
 def test_boxes_conversion():
     """Test the conversion of boxes between different modes.
 
-    ComandLine:
+    CommandLine:
         xdoctest tests/test_box3d.py::test_boxes_conversion zero
     """
     lidar_boxes = LiDARInstance3DBoxes(
@@ -1121,7 +1121,7 @@ def test_camera_boxes3d():
 def test_boxes3d_overlaps():
     """Test the iou calculation of boxes in different modes.
 
-    ComandLine:
+    CommandLine:
         xdoctest tests/test_box3d.py::test_boxes3d_overlaps zero
     """
     if not torch.cuda.is_available():

tools/create_data.py

@@ -191,7 +191,9 @@ def waymo_data_prep(root_path,
     """
     from tools.dataset_converters import waymo_converter as waymo
 
-    splits = ['training', 'validation', 'testing']
+    splits = [
+        'training', 'validation', 'testing', 'testing_3d_camera_only_detection'
+    ]
     for i, split in enumerate(splits):
         load_dir = osp.join(root_path, 'waymo_format', split)
         if split == 'validation':
@@ -203,7 +205,8 @@ def waymo_data_prep(root_path,
             save_dir,
             prefix=str(i),
             workers=workers,
-            test_mode=(split == 'testing'))
+            test_mode=(split
+                       in ['testing', 'testing_3d_camera_only_detection']))
         converter.convert()
     # Generate waymo infos
     out_dir = osp.join(out_dir, 'kitti_format')
@@ -212,14 +215,14 @@ def waymo_data_prep(root_path,
     info_train_path = osp.join(out_dir, f'{info_prefix}_infos_train.pkl')
     info_val_path = osp.join(out_dir, f'{info_prefix}_infos_val.pkl')
     info_trainval_path = osp.join(out_dir, f'{info_prefix}_infos_trainval.pkl')
-    update_pkl_infos('kitti', out_dir=out_dir, pkl_path=info_train_path)
-    update_pkl_infos('kitti', out_dir=out_dir, pkl_path=info_val_path)
-    update_pkl_infos('kitti', out_dir=out_dir, pkl_path=info_trainval_path)
+    update_pkl_infos('waymo', out_dir=out_dir, pkl_path=info_train_path)
+    update_pkl_infos('waymo', out_dir=out_dir, pkl_path=info_val_path)
+    update_pkl_infos('waymo', out_dir=out_dir, pkl_path=info_trainval_path)
     GTDatabaseCreater(
         'WaymoDataset',
         out_dir,
         info_prefix,
-        f'{out_dir}/{info_prefix}_infos_train.pkl',
+        f'{info_prefix}_infos_train.pkl',
         relative_path=False,
         with_mask=False,
         num_worker=workers).create()

tools/dataset_converters/create_gt_database.py

@@ -196,7 +196,8 @@ def create_groundtruth_database(dataset_class_name,
         file_client_args = dict(backend='disk')
         dataset_cfg.update(
             test_mode=False,
-            split='training',
+            data_prefix=dict(
+                pts='training/velodyne', img='', sweeps='training/velodyne'),
             modality=dict(
                 use_lidar=True,
                 use_depth=False,
@@ -407,7 +408,9 @@ class GTDatabaseCreater:
         image_idx = example['sample_idx']
         points = example['points'].tensor.numpy()
         gt_boxes_3d = annos['gt_bboxes_3d'].tensor.numpy()
-        names = annos['gt_names']
+        names = [
+            self.dataset.metainfo['CLASSES'][i] for i in annos['gt_labels_3d']
+        ]
         group_dict = dict()
         if 'group_ids' in annos:
             group_ids = annos['group_ids']
@@ -510,7 +513,8 @@ class GTDatabaseCreater:
             file_client_args = dict(backend='disk')
             dataset_cfg.update(
                 test_mode=False,
-                split='training',
+                data_prefix=dict(
+                    pts='training/velodyne_reduced', img='training/image_2'),
                 modality=dict(
                     use_lidar=True,
                     use_depth=False,
@@ -534,6 +538,9 @@ class GTDatabaseCreater:
         elif self.dataset_class_name == 'NuScenesDataset':
             dataset_cfg.update(
                 use_valid_flag=True,
+                data_prefix=dict(
+                    pts='samples/LIDAR_TOP', img='',
+                    sweeps='sweeps/LIDAR_TOP'),
                 pipeline=[
                     dict(
                         type='LoadPointsFromFile',
@@ -556,7 +563,10 @@ class GTDatabaseCreater:
             file_client_args = dict(backend='disk')
             dataset_cfg.update(
                 test_mode=False,
-                split='training',
+                data_prefix=dict(
+                    pts='training/velodyne',
+                    img='',
+                    sweeps='training/velodyne'),
                 modality=dict(
                     use_lidar=True,
                     use_depth=False,
@@ -577,8 +587,8 @@ class GTDatabaseCreater:
                         file_client_args=file_client_args)
                 ])
 
-        dataset = build_dataset(dataset_cfg)
-        self.pipeline = dataset.pipeline
+        self.dataset = build_dataset(dataset_cfg)
+        self.pipeline = self.dataset.pipeline
         if self.database_save_path is None:
             self.database_save_path = osp.join(
                 self.data_path, f'{self.info_prefix}_gt_database')
@@ -595,13 +605,15 @@ class GTDatabaseCreater:
                 self.file2id.update({info['file_name']: i})
 
         def loop_dataset(i):
-            input_dict = dataset.get_data_info(i)
-            dataset.pre_pipeline(input_dict)
+            input_dict = self.dataset.get_data_info(i)
+            input_dict['box_type_3d'] = self.dataset.box_type_3d
+            input_dict['box_mode_3d'] = self.dataset.box_mode_3d
             return input_dict
 
         multi_db_infos = mmengine.track_parallel_progress(
             self.create_single,
-            ((loop_dataset(i) for i in range(len(dataset))), len(dataset)),
+            ((loop_dataset(i)
+              for i in range(len(self.dataset))), len(self.dataset)),
             self.num_worker)
         print('Make global unique group id')
         group_counter_offset = 0

tools/dataset_converters/kitti_data_utils.py

@@ -46,8 +46,9 @@ def get_image_path(idx,
                    relative_path=True,
                    exist_check=True,
                    info_type='image_2',
+                   file_tail='.png',
                    use_prefix_id=False):
-    return get_kitti_info_path(idx, prefix, info_type, '.png', training,
+    return get_kitti_info_path(idx, prefix, info_type, file_tail, training,
                                relative_path, exist_check, use_prefix_id)
 
 
@@ -378,6 +379,7 @@ class WaymoInfoGatherer:
             self.training,
             self.relative_path,
             info_type='image_0',
+            file_tail='.jpg',
             use_prefix_id=True)
         if self.with_imageshape:
             img_path = image_info['image_path']
@@ -394,9 +396,18 @@ class WaymoInfoGatherer:
                 self.relative_path,
                 info_type='label_all',
                 use_prefix_id=True)
+            cam_sync_label_path = get_label_path(
+                idx,
+                self.path,
+                self.training,
+                self.relative_path,
+                info_type='cam_sync_label_all',
+                use_prefix_id=True)
             if self.relative_path:
                 label_path = str(root_path / label_path)
+                cam_sync_label_path = str(root_path / cam_sync_label_path)
             annotations = get_label_anno(label_path)
+            cam_sync_annotations = get_label_anno(cam_sync_label_path)
         info['image'] = image_info
         info['point_cloud'] = pc_info
         if self.calib:
@@ -434,11 +445,28 @@ class WaymoInfoGatherer:
             else:
                 rect_4x4 = R0_rect
 
+            # TODO: naming Tr_velo_to_cam or Tr_velo_to_cam0
             Tr_velo_to_cam = np.array([
                 float(info) for info in lines[6].split(' ')[1:13]
             ]).reshape([3, 4])
+            Tr_velo_to_cam1 = np.array([
+                float(info) for info in lines[7].split(' ')[1:13]
+            ]).reshape([3, 4])
+            Tr_velo_to_cam2 = np.array([
+                float(info) for info in lines[8].split(' ')[1:13]
+            ]).reshape([3, 4])
+            Tr_velo_to_cam3 = np.array([
+                float(info) for info in lines[9].split(' ')[1:13]
+            ]).reshape([3, 4])
+            Tr_velo_to_cam4 = np.array([
+                float(info) for info in lines[10].split(' ')[1:13]
+            ]).reshape([3, 4])
             if self.extend_matrix:
                 Tr_velo_to_cam = _extend_matrix(Tr_velo_to_cam)
+                Tr_velo_to_cam1 = _extend_matrix(Tr_velo_to_cam1)
+                Tr_velo_to_cam2 = _extend_matrix(Tr_velo_to_cam2)
+                Tr_velo_to_cam3 = _extend_matrix(Tr_velo_to_cam3)
+                Tr_velo_to_cam4 = _extend_matrix(Tr_velo_to_cam4)
             calib_info['P0'] = P0
             calib_info['P1'] = P1
             calib_info['P2'] = P2
@@ -446,7 +474,12 @@ class WaymoInfoGatherer:
             calib_info['P4'] = P4
             calib_info['R0_rect'] = rect_4x4
             calib_info['Tr_velo_to_cam'] = Tr_velo_to_cam
+            calib_info['Tr_velo_to_cam1'] = Tr_velo_to_cam1
+            calib_info['Tr_velo_to_cam2'] = Tr_velo_to_cam2
+            calib_info['Tr_velo_to_cam3'] = Tr_velo_to_cam3
+            calib_info['Tr_velo_to_cam4'] = Tr_velo_to_cam4
             info['calib'] = calib_info
+
         if self.pose:
             pose_path = get_pose_path(
                 idx,
@@ -460,6 +493,13 @@ class WaymoInfoGatherer:
             info['annos'] = annotations
             info['annos']['camera_id'] = info['annos'].pop('score')
             add_difficulty_to_annos(info)
+            info['cam_sync_annos'] = cam_sync_annotations
+            # NOTE: the 2D labels do not have strict correspondence with
+            # the projected 2D lidar labels
+            # e.g.: the projected 2D labels can be in camera 2
+            # while the most_visible_camera can have id 4
+            info['cam_sync_annos']['camera_id'] = info['cam_sync_annos'].pop(
+                'score')
 
         sweeps = []
         prev_idx = idx
@@ -484,6 +524,14 @@ class WaymoInfoGatherer:
                             relative_path=False,
                             use_prefix_id=True)) as f:
                     prev_info['timestamp'] = np.int64(f.read())
+                prev_info['image_path'] = get_image_path(
+                    prev_idx,
+                    self.path,
+                    self.training,
+                    self.relative_path,
+                    info_type='image_0',
+                    file_tail='.jpg',
+                    use_prefix_id=True)
                 prev_pose_path = get_pose_path(
                     prev_idx,
                     self.path,

tools/dataset_converters/nuscenes_converter.py

@@ -177,6 +177,7 @@ def _fill_trainval_infos(nusc,
 
         info = {
             'lidar_path': lidar_path,
+            'num_features': 5,
             'token': sample['token'],
             'sweeps': [],
             'cams': dict(),

tools/dataset_converters/update_infos_to_v2.py

@@ -3,6 +3,7 @@
 
 Example:
     python tools/dataset_converters/update_infos_to_v2.py
+        --dataset kitti
         --pkl ./data/kitti/kitti_infos_train.pkl
         --out-dir ./kitti_v2/
 """
@@ -16,8 +17,9 @@ import mmengine
 import numpy as np
 from nuscenes.nuscenes import NuScenes
 
-from mmdet3d.datasets.convert_utils import (convert_annos, get_2d_boxes,
-                                            get_waymo_2d_boxes)
+from mmdet3d.datasets.convert_utils import (convert_annos,
+                                            get_kitti_style_2d_boxes,
+                                            get_nuscenes_2d_boxes)
 from mmdet3d.datasets.utils import convert_quaternion_to_matrix
 from mmdet3d.structures import points_cam2img
 
@@ -194,7 +196,7 @@ def clear_data_info_unused_keys(data_info):
     empty_flag = True
     for key in keys:
         # we allow no annotations in datainfo
-        if key == 'instances':
+        if key in ['instances', 'cam_sync_instances', 'cam_instances']:
             empty_flag = False
             continue
         if isinstance(data_info[key], list):
@@ -217,7 +219,7 @@ def clear_data_info_unused_keys(data_info):
     return data_info, empty_flag
 
 
-def generate_camera_instances(info, nusc):
+def generate_nuscenes_camera_instances(info, nusc):
 
     # get bbox annotations for camera
     camera_types = [
@@ -234,7 +236,7 @@ def generate_camera_instances(info, nusc):
     for cam in camera_types:
         cam_info = info['cams'][cam]
         # list[dict]
-        ann_infos = get_2d_boxes(
+        ann_infos = get_nuscenes_2d_boxes(
             nusc,
             cam_info['sample_data_token'],
             visibilities=['', '1', '2', '3', '4'])
@@ -283,6 +285,8 @@ def update_nuscenes_infos(pkl_path, out_dir):
         temp_data_info['ego2global'] = convert_quaternion_to_matrix(
             ori_info_dict['ego2global_rotation'],
             ori_info_dict['ego2global_translation'])
+        temp_data_info['lidar_points']['num_pts_feats'] = ori_info_dict.get(
+            'num_features', 5)
         temp_data_info['lidar_points']['lidar_path'] = ori_info_dict[
             'lidar_path'].split('/')[-1]
         temp_data_info['lidar_points'][
@@ -355,7 +359,7 @@ def update_nuscenes_infos(pkl_path, out_dir):
             empty_instance['bbox_3d_isvalid'] = ori_info_dict['valid_flag'][i]
             empty_instance = clear_instance_unused_keys(empty_instance)
             temp_data_info['instances'].append(empty_instance)
-        temp_data_info['cam_instances'] = generate_camera_instances(
+        temp_data_info['cam_instances'] = generate_nuscenes_camera_instances(
             ori_info_dict, nusc)
         temp_data_info, _ = clear_data_info_unused_keys(temp_data_info)
         converted_list.append(temp_data_info)
@@ -485,6 +489,8 @@ def update_kitti_infos(pkl_path, out_dir):
             empty_instance = clear_instance_unused_keys(empty_instance)
             instance_list.append(empty_instance)
         temp_data_info['instances'] = instance_list
+        cam_instances = generate_kitti_camera_instances(ori_info_dict)
+        temp_data_info['cam_instances'] = cam_instances
         temp_data_info, _ = clear_data_info_unused_keys(temp_data_info)
         converted_list.append(temp_data_info)
     pkl_name = pkl_path.split('/')[-1]
@@ -510,7 +516,7 @@ def update_s3dis_infos(pkl_path, out_dir):
     converted_list = []
     for i, ori_info_dict in enumerate(mmengine.track_iter_progress(data_list)):
         temp_data_info = get_empty_standard_data_info()
-        temp_data_info['sample_id'] = i
+        temp_data_info['sample_idx'] = i
         temp_data_info['lidar_points']['num_pts_feats'] = ori_info_dict[
             'point_cloud']['num_features']
         temp_data_info['lidar_points']['lidar_path'] = ori_info_dict[
@@ -825,7 +831,7 @@ def update_waymo_infos(pkl_path, out_dir):
 
         if 'plane' in ori_info_dict:
             temp_data_info['plane'] = ori_info_dict['plane']
-        temp_data_info['sample_id'] = ori_info_dict['image']['image_idx']
+        temp_data_info['sample_idx'] = ori_info_dict['image']['image_idx']
 
         # calib matrix
         for cam_idx, cam_key in enumerate(camera_types):
@@ -995,6 +1001,18 @@ def update_waymo_infos(pkl_path, out_dir):
     mmengine.dump(converted_data_info, out_path, 'pkl')
 
 
+def generate_kitti_camera_instances(ori_info_dict):
+
+    cam_key = 'CAM2'
+    empty_camera_instances = get_empty_multicamera_instances([cam_key])
+    annos = copy.deepcopy(ori_info_dict['annos'])
+    ann_infos = get_kitti_style_2d_boxes(
+        ori_info_dict, occluded=[0, 1, 2, 3], annos=annos)
+    empty_camera_instances[cam_key] = ann_infos
+
+    return empty_camera_instances
+
+
 def generate_waymo_camera_instances(ori_info_dict, cam_keys):
 
     empty_multicamera_instances = get_empty_multicamera_instances(cam_keys)
@@ -1004,8 +1022,8 @@ def generate_waymo_camera_instances(ori_info_dict, cam_keys):
         if cam_idx != 0:
             annos = convert_annos(ori_info_dict, cam_idx)
 
-        ann_infos = get_waymo_2d_boxes(
-            ori_info_dict, cam_idx, occluded=[0], annos=annos)
+        ann_infos = get_kitti_style_2d_boxes(
+            ori_info_dict, cam_idx, occluded=[0], annos=annos, dataset='waymo')
 
         empty_multicamera_instances[cam_key] = ann_infos
     return empty_multicamera_instances
@@ -1017,7 +1035,7 @@ def parse_args():
     parser.add_argument(
         '--dataset', type=str, default='kitti', help='name of dataset')
     parser.add_argument(
-        '--pkl',
+        '--pkl-path',
         type=str,
         default='./data/kitti/kitti_infos_train.pkl ',
         help='specify the root dir of dataset')
@@ -1055,4 +1073,4 @@ if __name__ == '__main__':
     if args.out_dir is None:
         args.out_dir = args.root_dir
     update_pkl_infos(
-        dataset=args.dataset, out_dir=args.out_dir, pkl_path=args.pkl_path)
+        dataset=args.dataset, out_dir=args.out_dir, pkl_path=args.pkl)

tools/dataset_converters/waymo_converter.py

@@ -6,9 +6,8 @@ r"""Adapted from `Waymo to KITTI converter
 try:
     from waymo_open_dataset import dataset_pb2
 except ImportError:
-    raise ImportError(
-        'Please run "pip install waymo-open-dataset-tf-2-1-0==1.2.0" '
-        'to install the official devkit first.')
+    raise ImportError('Please run "pip install waymo-open-dataset-tf-2-5-0" '
+                      '>1.4.5 to install the official devkit first.')
 
 from glob import glob
 from os.path import join
@@ -34,7 +33,11 @@ class Waymo2KITTI(object):
         prefix (str): Prefix of filename. In general, 0 for training, 1 for
             validation and 2 for testing.
         workers (int, optional): Number of workers for the parallel process.
-        test_mode (bool, optional): Whether in the test_mode. Default: False.
+            Defaults to 64.
+        test_mode (bool, optional): Whether in the test_mode.
+            Defaults to False.
+        save_cam_sync_labels (bool, optional): Whether to save cam sync labels.
+            Defaults to True.
     """
 
     def __init__(self,
@@ -42,7 +45,8 @@ class Waymo2KITTI(object):
                  save_dir,
                  prefix,
                  workers=64,
-                 test_mode=False):
+                 test_mode=False,
+                 save_cam_sync_labels=True):
         self.filter_empty_3dboxes = True
         self.filter_no_label_zone_points = True
 
@@ -58,6 +62,14 @@ class Waymo2KITTI(object):
         if int(tf.__version__.split('.')[0]) < 2:
             tf.enable_eager_execution()
 
+        # keep the order defined by the official protocol
+        self.cam_list = [
+            '_FRONT',
+            '_FRONT_LEFT',
+            '_FRONT_RIGHT',
+            '_SIDE_LEFT',
+            '_SIDE_RIGHT',
+        ]
         self.lidar_list = [
             '_FRONT', '_FRONT_RIGHT', '_FRONT_LEFT', '_SIDE_RIGHT',
             '_SIDE_LEFT'
@@ -78,6 +90,7 @@ class Waymo2KITTI(object):
         self.prefix = prefix
         self.workers = int(workers)
         self.test_mode = test_mode
+        self.save_cam_sync_labels = save_cam_sync_labels
 
         self.tfrecord_pathnames = sorted(
             glob(join(self.load_dir, '*.tfrecord')))
@@ -89,6 +102,10 @@ class Waymo2KITTI(object):
         self.point_cloud_save_dir = f'{self.save_dir}/velodyne'
         self.pose_save_dir = f'{self.save_dir}/pose'
         self.timestamp_save_dir = f'{self.save_dir}/timestamp'
+        if self.save_cam_sync_labels:
+            self.cam_sync_label_save_dir = f'{self.save_dir}/cam_sync_label_'
+            self.cam_sync_label_all_save_dir = \
+                f'{self.save_dir}/cam_sync_label_all'
 
         self.create_folder()
 
@@ -124,14 +141,17 @@ class Waymo2KITTI(object):
             self.save_timestamp(frame, file_idx, frame_idx)
 
             if not self.test_mode:
+                # TODO save the depth image for waymo challenge solution.
                 self.save_label(frame, file_idx, frame_idx)
+                if self.save_cam_sync_labels:
+                    self.save_label(frame, file_idx, frame_idx, cam_sync=True)
 
     def __len__(self):
         """Length of the filename list."""
         return len(self.tfrecord_pathnames)
 
     def save_image(self, frame, file_idx, frame_idx):
-        """Parse and save the images in png format.
+        """Parse and save the images in jpg format.
 
         Args:
             frame (:obj:`Frame`): Open dataset frame proto.
@@ -141,7 +161,7 @@ class Waymo2KITTI(object):
         for img in frame.images:
             img_path = f'{self.image_save_dir}{str(img.name - 1)}/' + \
                 f'{self.prefix}{str(file_idx).zfill(3)}' + \
-                f'{str(frame_idx).zfill(3)}.png'
+                f'{str(frame_idx).zfill(3)}.jpg'
             img = mmcv.imfrombytes(img.image)
             mmcv.imwrite(img, img_path)
 
@@ -209,7 +229,7 @@ class Waymo2KITTI(object):
             file_idx (int): Current file index.
             frame_idx (int): Current frame index.
         """
-        range_images, camera_projections, range_image_top_pose = \
+        range_images, camera_projections, seg_labels, range_image_top_pose = \
             parse_range_image_and_camera_projection(frame)
 
         # First return
@@ -255,7 +275,7 @@ class Waymo2KITTI(object):
             f'{str(file_idx).zfill(3)}{str(frame_idx).zfill(3)}.bin'
         point_cloud.astype(np.float32).tofile(pc_path)
 
-    def save_label(self, frame, file_idx, frame_idx):
+    def save_label(self, frame, file_idx, frame_idx, cam_sync=False):
         """Parse and save the label data in txt format.
         The relation between waymo and kitti coordinates is noteworthy:
         1. x, y, z correspond to l, w, h (waymo) -> l, h, w (kitti)
@@ -267,10 +287,15 @@ class Waymo2KITTI(object):
             frame (:obj:`Frame`): Open dataset frame proto.
             file_idx (int): Current file index.
             frame_idx (int): Current frame index.
+            cam_sync (bool, optional): Whether to save the cam sync labels.
+                Defaults to False.
         """
-        fp_label_all = open(
-            f'{self.label_all_save_dir}/{self.prefix}' +
-            f'{str(file_idx).zfill(3)}{str(frame_idx).zfill(3)}.txt', 'w+')
+        label_all_path = f'{self.label_all_save_dir}/{self.prefix}' + \
+            f'{str(file_idx).zfill(3)}{str(frame_idx).zfill(3)}.txt'
+        if cam_sync:
+            label_all_path = label_all_path.replace('label_',
+                                                    'cam_sync_label_')
+        fp_label_all = open(label_all_path, 'w+')
         id_to_bbox = dict()
         id_to_name = dict()
         for labels in frame.projected_lidar_labels:
@@ -296,6 +321,21 @@ class Waymo2KITTI(object):
                     name = str(id_to_name.get(id + lidar))
                     break
 
+            # NOTE: the 2D labels do not have strict correspondence with
+            # the projected 2D lidar labels
+            # e.g.: the projected 2D labels can be in camera 2
+            # while the most_visible_camera can have id 4
+            if cam_sync:
+                if obj.most_visible_camera_name:
+                    name = str(
+                        self.cam_list.index(
+                            f'_{obj.most_visible_camera_name}'))
+                    box3d = obj.camera_synced_box
+                else:
+                    continue
+            else:
+                box3d = obj.box
+
             if bounding_box is None or name is None:
                 name = '0'
                 bounding_box = (0, 0, 0, 0)
@@ -310,20 +350,20 @@ class Waymo2KITTI(object):
 
             my_type = self.waymo_to_kitti_class_map[my_type]
 
-            height = obj.box.height
-            width = obj.box.width
-            length = obj.box.length
+            height = box3d.height
+            width = box3d.width
+            length = box3d.length
 
-            x = obj.box.center_x
-            y = obj.box.center_y
-            z = obj.box.center_z - height / 2
+            x = box3d.center_x
+            y = box3d.center_y
+            z = box3d.center_z - height / 2
 
             # project bounding box to the virtual reference frame
             pt_ref = self.T_velo_to_front_cam @ \
                 np.array([x, y, z, 1]).reshape((4, 1))
             x, y, z, _ = pt_ref.flatten().tolist()
 
-            rotation_y = -obj.box.heading - np.pi / 2
+            rotation_y = -box3d.heading - np.pi / 2
             track_id = obj.id
 
             # not available
@@ -345,9 +385,11 @@ class Waymo2KITTI(object):
             else:
                 line_all = line[:-1] + ' ' + name + '\n'
 
-            fp_label = open(
-                f'{self.label_save_dir}{name}/{self.prefix}' +
-                f'{str(file_idx).zfill(3)}{str(frame_idx).zfill(3)}.txt', 'a')
+            label_path = f'{self.label_save_dir}{name}/{self.prefix}' + \
+                f'{str(file_idx).zfill(3)}{str(frame_idx).zfill(3)}.txt'
+            if cam_sync:
+                label_path = label_path.replace('label_', 'cam_sync_label_')
+            fp_label = open(label_path, 'a')
             fp_label.write(line)
             fp_label.close()
 
@@ -398,11 +440,16 @@ class Waymo2KITTI(object):
         """Create folder for data preprocessing."""
         if not self.test_mode:
             dir_list1 = [
-                self.label_all_save_dir, self.calib_save_dir,
-                self.point_cloud_save_dir, self.pose_save_dir,
-                self.timestamp_save_dir
+                self.label_all_save_dir,
+                self.calib_save_dir,
+                self.point_cloud_save_dir,
+                self.pose_save_dir,
+                self.timestamp_save_dir,
             ]
             dir_list2 = [self.label_save_dir, self.image_save_dir]
+            if self.save_cam_sync_labels:
+                dir_list1.append(self.cam_sync_label_all_save_dir)
+                dir_list2.append(self.cam_sync_label_save_dir)
         else:
             dir_list1 = [
                 self.calib_save_dir, self.point_cloud_save_dir,

tools/dist_train.sh

@@ -16,5 +16,4 @@ python -m torch.distributed.launch \
     --master_port=$PORT \
     $(dirname "$0")/train.py \
     $CONFIG \
-    --seed 0 \
     --launcher pytorch ${@:3}

tools/misc/browse_dataset.py

@@ -2,42 +2,41 @@
 import argparse
 from os import path as osp
 
-import mmengine
-from mmengine import Config, DictAction, mkdir_or_exist
+from mmengine.config import Config, DictAction
+from mmengine.utils import ProgressBar, mkdir_or_exist
 
-from mmdet3d.datasets import build_dataset
-from mmdet3d.registry import VISUALIZERS
-from mmdet3d.utils import register_all_modules
+from mmdet3d.registry import DATASETS, VISUALIZERS
+from mmdet3d.utils import register_all_modules, replace_ceph_backend
 
 
 def parse_args():
     parser = argparse.ArgumentParser(description='Browse a dataset')
     parser.add_argument('config', help='train config file path')
-    parser.add_argument(
-        '--skip-type',
-        type=str,
-        nargs='+',
-        default=['Normalize'],
-        help='skip some useless pipeline')
     parser.add_argument(
         '--output-dir',
         default=None,
         type=str,
         help='If there is no display interface, you can save it')
+    parser.add_argument('--not-show', default=False, action='store_true')
+    parser.add_argument(
+        '--show-interval',
+        type=float,
+        default=2,
+        help='the interval of show (s)')
     parser.add_argument(
         '--task',
         type=str,
-        choices=['det', 'seg', 'multi_modality-det', 'mono-det'],
+        choices=[
+            'mono_det', 'multi-view_det', 'lidar_det', 'lidar_seg',
+            'multi-modality_det'
+        ],
         help='Determine the visualization method depending on the task.')
     parser.add_argument(
         '--aug',
         action='store_true',
         help='Whether to visualize augmented datasets or original dataset.')
     parser.add_argument(
-        '--online',
-        action='store_true',
-        help='Whether to perform online visualization. Note that you often '
-        'need a monitor to do so.')
+        '--ceph', action='store_true', help='Use ceph as data storage backend')
     parser.add_argument(
         '--cfg-options',
         nargs='+',
@@ -52,20 +51,22 @@ def parse_args():
     return args
 
 
-def build_data_cfg(config_path, skip_type, aug, cfg_options):
+def build_data_cfg(config_path, aug, cfg_options):
     """Build data config for loading visualization data."""
 
     cfg = Config.fromfile(config_path)
     if cfg_options is not None:
         cfg.merge_from_dict(cfg_options)
-    # extract inner dataset of `RepeatDataset` as `cfg.data.train`
-    # so we don't need to worry about it later
-    if cfg.data.train['type'] == 'RepeatDataset':
-        cfg.data.train = cfg.data.train.dataset
+
+    # extract inner dataset of `RepeatDataset` as
+    # `cfg.train_dataloader.dataset` so we don't
+    # need to worry about it later
+    if cfg.train_dataloader.dataset['type'] == 'RepeatDataset':
+        cfg.train_dataloader.dataset = cfg.train_dataloader.dataset.dataset
     # use only first dataset for `ConcatDataset`
-    if cfg.data.train['type'] == 'ConcatDataset':
-        cfg.data.train = cfg.data.train.datasets[0]
-    train_data_cfg = cfg.data.train
+    if cfg.train_dataloader.dataset['type'] == 'ConcatDataset':
+        cfg.train_dataloader.dataset = cfg.train_dataloader.dataset.datasets[0]
+    train_data_cfg = cfg.train_dataloader.dataset
 
     if aug:
         show_pipeline = cfg.train_pipeline
@@ -74,16 +75,14 @@ def build_data_cfg(config_path, skip_type, aug, cfg_options):
         for i in range(len(cfg.train_pipeline)):
             if cfg.train_pipeline[i]['type'] == 'LoadAnnotations3D':
                 show_pipeline.insert(i, cfg.train_pipeline[i])
-            # Collect points as well as labels
-            if cfg.train_pipeline[i]['type'] == 'Pack3DInputs':
-                if show_pipeline[-1]['type'] == 'Pack3DInputs':
+            # Collect data as well as labels
+            if cfg.train_pipeline[i]['type'] == 'Pack3DDetInputs':
+                if show_pipeline[-1]['type'] == 'Pack3DDetInputs':
                     show_pipeline[-1] = cfg.train_pipeline[i]
                 else:
                     show_pipeline.append(cfg.train_pipeline[i])
 
-    train_data_cfg['pipeline'] = [
-        x for x in show_pipeline if x['type'] not in skip_type
-    ]
+    train_data_cfg['pipeline'] = show_pipeline
 
     return cfg
 
@@ -94,26 +93,29 @@ def main():
     if args.output_dir is not None:
         mkdir_or_exist(args.output_dir)
 
-    cfg = build_data_cfg(args.config, args.skip_type, args.aug,
-                         args.cfg_options)
+    cfg = build_data_cfg(args.config, args.aug, args.cfg_options)
+
+    # TODO: We will unify the ceph support approach with other OpenMMLab repos
+    if args.ceph:
+        cfg = replace_ceph_backend(cfg)
 
     # register all modules in mmdet3d into the registries
     register_all_modules()
 
     try:
-        dataset = build_dataset(
+        dataset = DATASETS.build(
             cfg.train_dataloader.dataset,
             default_args=dict(filter_empty_gt=False))
     except TypeError:  # seg dataset doesn't have `filter_empty_gt` key
-        dataset = build_dataset(cfg.train_dataloader.dataset)
+        dataset = DATASETS.build(cfg.train_dataloader.dataset)
 
     # configure visualization mode
-    vis_task = args.task  # 'det', 'seg', 'multi_modality-det', 'mono-det'
+    vis_task = args.task
 
     visualizer = VISUALIZERS.build(cfg.visualizer)
     visualizer.dataset_meta = dataset.metainfo
 
-    progress_bar = mmengine.ProgressBar(len(dataset))
+    progress_bar = ProgressBar(len(dataset))
 
     for item in dataset:
         # the 3D Boxes in input could be in any of three coordinates
@@ -126,7 +128,7 @@ def main():
         visualizer.add_datasample(
             '3d visualzier',
             data_input,
-            data_sample,
+            data_sample=data_sample,
             show=not args.not_show,
             wait_time=args.show_interval,
             out_file=out_file,