Update waymo dataset, evaluation metrics and related configs for 2.0 (#1663)

* update waymo dataset

* [Fix] Fix all unittests and refactor tests directory and add circle ci in `test-1.x` (#1654)

* add circle ci

* delete github ci

* fix ci

* fix ut

* fix markdown version

* rm

* fix part of uts

* fix comments

* change foler

* refactor test directory

* fix kitti metric ut

* fix all ut
Co-authored-by: VVsssssk <shenkun@pjlab.org.cn>

* add waymo dataset and evaluation metrics

* convert second configs for v2.0

* [Refactor] Unify ceph config (#1677)

* refactor ceph in config

* support metric load ann file from ceph

* add doc string and remove useless code

* [Fix]Fix create data (#1659)

* add circle ci

* delete github ci

* fix ci

* fix ut

* fix markdown version

* rm

* fix part of uts

* fix comments

* change foler

* refactor test directory

* fix kitti metric ut

* fix all ut

* fix creat data
Co-authored-by: ChaimZhu <zhuchenming@pjlab.org.cn>

* [Fix] Fix seg mapping (#1681)

* [Doc]: fix markdown version (#1653)

* [CI] Add circle ci (#1647)

* add circle ci

* delete github ci

* fix ci

* fix ut

* fix markdown version

* rm

* fix seg mapping for scannet

* fix requiremetn

* fix all seg dataet

* resolve commnets
Co-authored-by: VVsssssk <88368822+VVsssssk@users.noreply.github.com>

* [Fix] Fix SSN configs (#1686)

* modify doc string and evaluation file location

* add doc string

* remove path mapping in flieclient args
Co-authored-by: ChaimZhu <zhuchenming@pjlab.org.cn>
Co-authored-by: VVsssssk <shenkun@pjlab.org.cn>
Co-authored-by: VVsssssk <88368822+VVsssssk@users.noreply.github.com>
Co-authored-by: Shilong Zhang <61961338+jshilong@users.noreply.github.com>

tests/test_data/test_transforms/test_loading.py → tests/test_datasets/test_transforms/test_loading.py

@@ -80,8 +80,10 @@ class TestPointSegClassMapping(unittest.TestCase):
     def test_point_seg_class_mapping(self):
         results = dict()
         results['pts_semantic_mask'] = np.array([1, 2, 3, 4, 5])
-        results['label_mapping'] = {0: 3, 1: 0, 2: 1, 3: 2, 4: 3, 5: 3}
-        point_seg_mapping_transform = PointSegClassMapping()
+
+        point_seg_mapping_transform = PointSegClassMapping(
+            valid_cat_ids=[1, 2, 3],
+            max_cat_id=results['pts_semantic_mask'].max())
         results = point_seg_mapping_transform(results)
         assert_allclose(results['pts_semantic_mask'], np.array([0, 1, 2, 3,
                                                                 3]))

tests/test_metrics/test_instance_seg_metric.py → tests/test_evaluation/test_metrics/test_instance_seg_metric.py

@@ -14,7 +14,6 @@ class TestInstanceSegMetric(unittest.TestCase):
     def _demo_mm_inputs(self):
         """Create a superset of inputs needed to run test or train batches."""
         packed_inputs = []
-        results_dict = dict()
         mm_inputs = dict()
         n_points = 3300
         gt_labels = [0, 0, 0, 0, 0, 0, 14, 14, 2, 1]
@@ -26,12 +25,11 @@ class TestInstanceSegMetric(unittest.TestCase):
             gt_instance_mask[begin:end] = i
             gt_semantic_mask[begin:end] = gt_label
 
-        results_dict['pts_instance_mask'] = torch.tensor(gt_instance_mask)
-        results_dict['pts_semantic_mask'] = torch.tensor(gt_semantic_mask)
+        ann_info_data = dict()
+        ann_info_data['pts_instance_mask'] = torch.tensor(gt_instance_mask)
+        ann_info_data['pts_semantic_mask'] = torch.tensor(gt_semantic_mask)
 
-        data_sample = Det3DDataSample()
-        data_sample.gt_pts_seg = PointData(**results_dict)
-        mm_inputs['data_sample'] = data_sample.to_dict()
+        mm_inputs['data_sample'] = dict(eval_ann_info=ann_info_data)
         packed_inputs.append(mm_inputs)
 
         return packed_inputs
@@ -80,5 +78,5 @@ class TestInstanceSegMetric(unittest.TestCase):
         instance_seg_metric = InstanceSegMetric()
         instance_seg_metric.dataset_meta = dataset_meta
         instance_seg_metric.process(data_batch, predictions)
-        res = instance_seg_metric.evaluate(6)
+        res = instance_seg_metric.evaluate(1)
         self.assertIsInstance(res, dict)

tests/test_metrics/test_kitti_metric.py → tests/test_evaluation/test_metrics/test_kitti_metric.py

@@ -50,7 +50,7 @@ def test_multi_modal_kitti_metric():
         pytest.skip('test requires GPU and torch+cuda')
     kittimetric = KittiMetric(
         data_root + '/kitti_infos_train.pkl', metric=['mAP'])
-    kittimetric.dataset_meta = dict(CLASSES=['Car', 'Pedestrian', 'Cyclist'])
+    kittimetric.dataset_meta = dict(CLASSES=['Pedestrian', 'Cyclist', 'Car'])
     data_batch, predictions = _init_multi_modal_evaluate_input()
     kittimetric.process(data_batch, predictions)
     ap_dict = kittimetric.compute_metrics(kittimetric.results)
@@ -73,7 +73,7 @@ def test_kitti_metric_mAP():
         pytest.skip('test requires GPU and torch+cuda')
     kittimetric = KittiMetric(
         data_root + '/kitti_infos_train.pkl', metric=['mAP'])
-    kittimetric.dataset_meta = dict(CLASSES=['Car', 'Pedestrian', 'Cyclist'])
+    kittimetric.dataset_meta = dict(CLASSES=['Pedestrian', 'Cyclist', 'Car'])
     data_batch, predictions = _init_evaluate_input()
     kittimetric.process(data_batch, predictions)
     ap_dict = kittimetric.compute_metrics(kittimetric.results)

tests/test_metrics/test_seg_metric.py → tests/test_evaluation/test_metrics/test_seg_metric.py

@@ -14,27 +14,24 @@ class TestSegMetric(unittest.TestCase):
         """Create a superset of inputs needed to run test or train batches."""
         packed_inputs = []
         mm_inputs = dict()
-        data_sample = Det3DDataSample()
         pts_semantic_mask = torch.Tensor([
             0, 0, 0, 255, 0, 0, 1, 1, 1, 255, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3,
             3, 255
         ])
-        gt_pts_seg_data = dict(pts_semantic_mask=pts_semantic_mask)
-        data_sample.gt_pts_seg = PointData(**gt_pts_seg_data)
-        mm_inputs['data_sample'] = data_sample.to_dict()
+        ann_info_data = dict(pts_semantic_mask=pts_semantic_mask)
+        mm_inputs['data_sample'] = dict(eval_ann_info=ann_info_data)
         packed_inputs.append(mm_inputs)
 
         return packed_inputs
 
     def _demo_mm_model_output(self):
         """Create a superset of inputs needed to run test or train batches."""
-        results_dict = dict()
-        pts_seg_pred = torch.Tensor([
+        pts_semantic_mask = torch.Tensor([
             0, 0, 1, 0, 0, 2, 1, 3, 1, 2, 1, 0, 2, 2, 2, 2, 1, 3, 0, 3, 3, 3, 3
         ])
-        results_dict['pts_semantic_mask'] = pts_seg_pred
+        pred_pts_seg_data = dict(pts_semantic_mask=pts_semantic_mask)
         data_sample = Det3DDataSample()
-        data_sample['pred_pts_seg'] = results_dict
+        data_sample.pred_pts_seg = PointData(**pred_pts_seg_data)
         batch_data_samples = [data_sample]
 
         predictions = []
@@ -58,5 +55,5 @@ class TestSegMetric(unittest.TestCase):
         seg_metric = SegMetric()
         seg_metric.dataset_meta = dataset_meta
         seg_metric.process(data_batch, predictions)
-        res = seg_metric.evaluate(0)
+        res = seg_metric.evaluate(1)
         self.assertIsInstance(res, dict)

tests/test_models/test_decode_heads/test_dgcnn_head.py

@@ -34,17 +34,17 @@ class TestDGCNNHead(TestCase):
         # Test forward
         seg_logits = dgcnn_head.forward(feat_dict)
 
-        self.assertEqual(seg_logits, torch.Size([1, 13, 4096]))
+        self.assertEqual(seg_logits.shape, torch.Size([1, 13, 4096]))
 
         # When truth is non-empty then losses
         # should be nonzero for random inputs
-        pts_semantic_mask = torch.randint(0, 13, (2, 4096)).long()
+        pts_semantic_mask = torch.randint(0, 13, (4096, )).long()
         gt_pts_seg = PointData(pts_semantic_mask=pts_semantic_mask)
 
         datasample = Det3DDataSample()
         datasample.gt_pts_seg = gt_pts_seg
 
-        gt_losses = dgcnn_head.loss(seg_logits, [datasample])
+        gt_losses = dgcnn_head.loss_by_feat(seg_logits, [datasample])
 
         gt_sem_seg_loss = gt_losses['loss_sem_seg'].item()
 

tests/test_models/test_decode_heads/test_paconv_head.py

@@ -12,55 +12,57 @@ class TestPAConvHead(TestCase):
     def test_paconv_head_loss(self):
         """Tests PAConv head loss."""
 
-        paconv_head = PAConvHead(
-            fp_channels=((768, 256, 256), (384, 256, 256), (320, 256, 128),
-                         (128 + 6, 128, 128, 128)),
-            channels=128,
-            num_classes=20,
-            dropout_ratio=0.5,
-            conv_cfg=dict(type='Conv1d'),
-            norm_cfg=dict(type='BN1d'),
-            act_cfg=dict(type='ReLU'),
-            loss_decode=dict(
-                type='mmdet.CrossEntropyLoss',
-                use_sigmoid=False,
-                class_weight=None,
-                loss_weight=1.0),
-            ignore_index=20)
+        if torch.cuda.is_available():
+            paconv_head = PAConvHead(
+                fp_channels=((768, 256, 256), (384, 256, 256), (320, 256, 128),
+                             (128 + 6, 128, 128, 128)),
+                channels=128,
+                num_classes=20,
+                dropout_ratio=0.5,
+                conv_cfg=dict(type='Conv1d'),
+                norm_cfg=dict(type='BN1d'),
+                act_cfg=dict(type='ReLU'),
+                loss_decode=dict(
+                    type='mmdet.CrossEntropyLoss',
+                    use_sigmoid=False,
+                    class_weight=None,
+                    loss_weight=1.0),
+                ignore_index=20)
 
-        # PAConv head expects dict format features
-        sa_xyz = [
-            torch.rand(1, 4096, 3).float(),
-            torch.rand(1, 1024, 3).float(),
-            torch.rand(1, 256, 3).float(),
-            torch.rand(1, 64, 3).float(),
-            torch.rand(1, 16, 3).float(),
-        ]
-        sa_features = [
-            torch.rand(1, 6, 4096).float(),
-            torch.rand(1, 64, 1024).float(),
-            torch.rand(1, 128, 256).float(),
-            torch.rand(1, 256, 64).float(),
-            torch.rand(1, 512, 16).float(),
-        ]
-        feat_dict = dict(sa_xyz=sa_xyz, sa_features=sa_features)
+            paconv_head.cuda()
+            # PAConv head expects dict format features
+            sa_xyz = [
+                torch.rand(1, 4096, 3).float().cuda(),
+                torch.rand(1, 1024, 3).float().cuda(),
+                torch.rand(1, 256, 3).float().cuda(),
+                torch.rand(1, 64, 3).float().cuda(),
+                torch.rand(1, 16, 3).float().cuda(),
+            ]
+            sa_features = [
+                torch.rand(1, 6, 4096).float().cuda(),
+                torch.rand(1, 64, 1024).float().cuda(),
+                torch.rand(1, 128, 256).float().cuda(),
+                torch.rand(1, 256, 64).float().cuda(),
+                torch.rand(1, 512, 16).float().cuda(),
+            ]
+            feat_dict = dict(sa_xyz=sa_xyz, sa_features=sa_features)
 
-        # Test forward
-        seg_logits = paconv_head.forward(feat_dict)
+            # Test forward
+            seg_logits = paconv_head.forward(feat_dict)
 
-        self.assertEqual(seg_logits, torch.Size([1, 20, 4096]))
+            self.assertEqual(seg_logits.shape, torch.Size([1, 20, 4096]))
 
-        # When truth is non-empty then losses
-        # should be nonzero for random inputs
-        pts_semantic_mask = torch.randint(0, 20, (4096)).long()
-        gt_pts_seg = PointData(pts_semantic_mask=pts_semantic_mask)
+            # When truth is non-empty then losses
+            # should be nonzero for random inputs
+            pts_semantic_mask = torch.randint(0, 20, (4096, )).long().cuda()
+            gt_pts_seg = PointData(pts_semantic_mask=pts_semantic_mask)
 
-        datasample = Det3DDataSample()
-        datasample.gt_pts_seg = gt_pts_seg
+            datasample = Det3DDataSample()
+            datasample.gt_pts_seg = gt_pts_seg
 
-        gt_losses = paconv_head.loss(seg_logits, [datasample])
+            gt_losses = paconv_head.loss_by_feat(seg_logits, [datasample])
 
-        gt_sem_seg_loss = gt_losses['loss_sem_seg'].item()
+            gt_sem_seg_loss = gt_losses['loss_sem_seg'].item()
 
-        self.assertGreater(gt_sem_seg_loss, 0,
-                           'semantic seg loss should be positive')
+            self.assertGreater(gt_sem_seg_loss, 0,
+                               'semantic seg loss should be positive')

tests/test_models/test_decode_heads/test_pointnet2_head.py

@@ -12,55 +12,58 @@ class TestPointNet2Head(TestCase):
     def test_paconv_head_loss(self):
         """Tests PAConv head loss."""
 
-        pointnet2_head = PointNet2Head(
-            fp_channels=((768, 256, 256), (384, 256, 256), (320, 256, 128),
-                         (128, 128, 128, 128)),
-            channels=128,
-            num_classes=20,
-            dropout_ratio=0.5,
-            conv_cfg=dict(type='Conv1d'),
-            norm_cfg=dict(type='BN1d'),
-            act_cfg=dict(type='ReLU'),
-            loss_decode=dict(
-                type='mmdet.CrossEntropyLoss',
-                use_sigmoid=False,
-                class_weight=None,
-                loss_weight=1.0),
-            ignore_index=20)
+        if torch.cuda.is_available():
+            pointnet2_head = PointNet2Head(
+                fp_channels=((768, 256, 256), (384, 256, 256), (320, 256, 128),
+                             (128, 128, 128, 128)),
+                channels=128,
+                num_classes=20,
+                dropout_ratio=0.5,
+                conv_cfg=dict(type='Conv1d'),
+                norm_cfg=dict(type='BN1d'),
+                act_cfg=dict(type='ReLU'),
+                loss_decode=dict(
+                    type='mmdet.CrossEntropyLoss',
+                    use_sigmoid=False,
+                    class_weight=None,
+                    loss_weight=1.0),
+                ignore_index=20)
 
-        # DGCNN head expects dict format features
-        sa_xyz = [
-            torch.rand(1, 4096, 3).float(),
-            torch.rand(1, 1024, 3).float(),
-            torch.rand(1, 256, 3).float(),
-            torch.rand(1, 64, 3).float(),
-            torch.rand(1, 16, 3).float(),
-        ]
-        sa_features = [
-            torch.rand(1, 6, 4096).float(),
-            torch.rand(1, 64, 1024).float(),
-            torch.rand(1, 128, 256).float(),
-            torch.rand(1, 256, 64).float(),
-            torch.rand(1, 512, 16).float(),
-        ]
-        feat_dict = dict(sa_xyz=sa_xyz, sa_features=sa_features)
+            pointnet2_head.cuda()
 
-        # Test forward
-        seg_logits = pointnet2_head.forward(feat_dict)
+            # DGCNN head expects dict format features
+            sa_xyz = [
+                torch.rand(1, 4096, 3).float().cuda(),
+                torch.rand(1, 1024, 3).float().cuda(),
+                torch.rand(1, 256, 3).float().cuda(),
+                torch.rand(1, 64, 3).float().cuda(),
+                torch.rand(1, 16, 3).float().cuda(),
+            ]
+            sa_features = [
+                torch.rand(1, 6, 4096).float().cuda(),
+                torch.rand(1, 64, 1024).float().cuda(),
+                torch.rand(1, 128, 256).float().cuda(),
+                torch.rand(1, 256, 64).float().cuda(),
+                torch.rand(1, 512, 16).float().cuda(),
+            ]
+            feat_dict = dict(sa_xyz=sa_xyz, sa_features=sa_features)
 
-        self.assertEqual(seg_logits, torch.Size([1, 20, 4096]))
+            # Test forward
+            seg_logits = pointnet2_head.forward(feat_dict)
 
-        # When truth is non-empty then losses
-        # should be nonzero for random inputs
-        pts_semantic_mask = torch.randint(0, 20, (4096)).long()
-        gt_pts_seg = PointData(pts_semantic_mask=pts_semantic_mask)
+            self.assertEqual(seg_logits.shape, torch.Size([1, 20, 4096]))
 
-        datasample = Det3DDataSample()
-        datasample.gt_pts_seg = gt_pts_seg
+            # When truth is non-empty then losses
+            # should be nonzero for random inputs
+            pts_semantic_mask = torch.randint(0, 20, (4096, )).long().cuda()
+            gt_pts_seg = PointData(pts_semantic_mask=pts_semantic_mask)
 
-        gt_losses = pointnet2_head.loss(seg_logits, [datasample])
+            datasample = Det3DDataSample()
+            datasample.gt_pts_seg = gt_pts_seg
 
-        gt_sem_seg_loss = gt_losses['loss_sem_seg'].item()
+            gt_losses = pointnet2_head.loss_by_feat(seg_logits, [datasample])
 
-        self.assertGreater(gt_sem_seg_loss, 0,
-                           'semantic seg loss should be positive')
+            gt_sem_seg_loss = gt_losses['loss_sem_seg'].item()
+
+            self.assertGreater(gt_sem_seg_loss, 0,
+                               'semantic seg loss should be positive')

tests/test_models/test_detectors/test_imvoxelnet.py

@@ -48,6 +48,6 @@ class TestImVoxelNet(unittest.TestCase):
             with torch.no_grad():
                 losses = model.forward(batch_inputs, data_samples, mode='loss')
 
-            self.assertGreater(losses['loss_cls'], 0)
-            self.assertGreater(losses['loss_bbox'], 0)
-            self.assertGreater(losses['loss_dir'], 0)
+            self.assertGreaterEqual(losses['loss_cls'][0], 0)
+            self.assertGreaterEqual(losses['loss_bbox'][0], 0)
+            self.assertGreaterEqual(losses['loss_dir'][0], 0)

tests/test_models/test_fusion_layers/test_vote_fusion.py

@@ -12,9 +12,9 @@ from mmdet3d.models.layers.fusion_layers import VoteFusion
 
 def test_vote_fusion():
     img_meta = {
-        'ori_shape': (530, 730, 3),
-        'img_shape': (600, 826, 3),
-        'pad_shape': (608, 832, 3),
+        'ori_shape': (530, 730),
+        'img_shape': (600, 826),
+        'pad_shape': (608, 832),
         'scale_factor':
         torch.tensor([1.1315, 1.1321, 1.1315, 1.1321]),
         'flip':