[Fix] Fix benchmark (#2402)

* init commit

* set random seed for groupfree and imvotenet

* replace lyft by nus

* add fcos3d resnet34

* Revert "add fcos3d resnet34"

This reverts commit f3da3d91253cb9340e8e0a98b0b54efadf842f8d.

* replace waymo by kitti for pointpillars

.dev_scripts/benchmark_full_models.txt

@@ -16,12 +16,11 @@ configs/pgd/pgd_r101-caffe_fpn_head-gn_4xb3-4x_kitti-mono3d.py
 configs/point_rcnn/point-rcnn_8xb2_kitti-3d-3class.py
 configs/pointnet2/pointnet2_msg_2xb16-cosine-80e_s3dis-seg.py
 configs/pointnet2/pointnet2_msg_2xb16-cosine-250e_scannet-seg.py
-configs/pointpillars/pointpillars_hv_fpn_sbn-all_8xb2-2x_lyft-3d.py
-configs/pointpillars/pointpillars_hv_secfpn_sbn-all_16xb2-2x_waymoD5-3d-3class.py
+configs/pointpillars/pointpillars_hv_secfpn_8xb6-160e_kitti-3d-3class.py
 configs/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class.py
 configs/regnet/pointpillars_hv_regnet-1.6gf_fpn_sbn-all_8xb4-2x_nus-3d.py
 configs/second/second_hv_secfpn_8xb6-80e_kitti-3d-3class.py
 configs/second/second_hv_secfpn_8xb6-amp-80e_kitti-3d-3class.py
 configs/smoke/smoke_dla34_dlaneck_gn-all_4xb8-6x_kitti-mono3d.py
-configs/ssn/ssn_hv_secfpn_sbn-all_16xb2-2x_lyft-3d.py
+configs/ssn/ssn_hv_secfpn_sbn-all_16xb2-2x_nus-3d.py
 configs/votenet/votenet_8xb8_scannet-3d.py

.dev_scripts/benchmark_train_models.txt

@@ -6,8 +6,7 @@ configs/mvxnet/mvxnet_fpn_dv_second_secfpn_8xb2-80e_kitti-3d-3class.py
 configs/point_rcnn/point-rcnn_8xb2_kitti-3d-3class.py
 configs/pointnet2/pointnet2_msg_2xb16-cosine-80e_s3dis-seg.py
 configs/pointnet2/pointnet2_msg_2xb16-cosine-250e_scannet-seg.py
-configs/pointpillars/pointpillars_hv_fpn_sbn-all_8xb2-2x_lyft-3d.py
-configs/pointpillars/pointpillars_hv_secfpn_sbn-all_16xb2-2x_waymoD5-3d-3class.py
+configs/pointpillars/pointpillars_hv_secfpn_8xb6-160e_kitti-3d-3class.py
 configs/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class.py
 configs/second/second_hv_secfpn_8xb6-80e_kitti-3d-3class.py
 configs/second/second_hv_secfpn_8xb6-amp-80e_kitti-3d-3class.py

configs/_base_/datasets/lyft-3d-range100.py

@@ -8,6 +8,7 @@ class_names = [
 ]
 dataset_type = 'LyftDataset'
 data_root = 'data/lyft/'
+data_prefix = dict(pts='v1.01-train/lidar', img='', sweeps='v1.01-train/lidar')
 # Input modality for Lyft dataset, this is consistent with the submission
 # format which requires the information in input_modality.
 input_modality = dict(
@@ -101,35 +102,49 @@ eval_pipeline = [
     dict(type='Pack3DDetInputs', keys=['points'])
 ]
 
-data = dict(
-    samples_per_gpu=2,
-    workers_per_gpu=2,
-    train=dict(
+train_dataloader = dict(
+    batch_size=2,
+    num_workers=2,
+    persistent_workers=True,
+    sampler=dict(type='DefaultSampler', shuffle=True),
+    dataset=dict(
         type=dataset_type,
         data_root=data_root,
-        ann_file=data_root + 'lyft_infos_train.pkl',
+        ann_file='lyft_infos_train.pkl',
         pipeline=train_pipeline,
-        classes=class_names,
+        metainfo=dict(classes=class_names),
         modality=input_modality,
-        test_mode=False),
-    val=dict(
+        data_prefix=data_prefix,
+        test_mode=False,
+        box_type_3d='LiDAR',
+        backend_args=backend_args))
+val_dataloader = dict(
+    batch_size=1,
+    num_workers=1,
+    persistent_workers=True,
+    drop_last=False,
+    sampler=dict(type='DefaultSampler', shuffle=False),
+    dataset=dict(
         type=dataset_type,
         data_root=data_root,
-        ann_file=data_root + 'lyft_infos_val.pkl',
+        ann_file='lyft_infos_val.pkl',
         pipeline=test_pipeline,
-        classes=class_names,
+        metainfo=dict(classes=class_names),
         modality=input_modality,
-        test_mode=True),
-    test=dict(
-        type=dataset_type,
+        test_mode=True,
+        data_prefix=data_prefix,
+        box_type_3d='LiDAR',
+        backend_args=backend_args))
+test_dataloader = val_dataloader
+
+val_evaluator = dict(
+    type='LyftMetric',
     data_root=data_root,
-        ann_file=data_root + 'lyft_infos_test.pkl',
-        pipeline=test_pipeline,
-        classes=class_names,
-        modality=input_modality,
-        test_mode=True))
-# For Lyft dataset, we usually evaluate the model at the end of training.
-# Since the models are trained by 24 epochs by default, we set evaluation
-# interval to be 24. Please change the interval accordingly if you do not
-# use a default schedule.
-evaluation = dict(interval=24, pipeline=eval_pipeline)
+    ann_file='lyft_infos_val.pkl',
+    metric='bbox',
+    backend_args=backend_args)
+test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+    type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/lyft-3d.py

@@ -11,7 +11,7 @@ data_root = 'data/lyft/'
 # Input modality for Lyft dataset, this is consistent with the submission
 # format which requires the information in input_modality.
 input_modality = dict(use_lidar=True, use_camera=False)
-data_prefix = dict(pts='samples/LIDAR_TOP', img='', sweeps='sweeps/LIDAR_TOP')
+data_prefix = dict(pts='v1.01-train/lidar', img='', sweeps='v1.01-train/lidar')
 
 # Example to use different file client
 # Method 1: simply set the data root and let the file I/O module
@@ -149,14 +149,11 @@ val_dataloader = dict(
 
 val_evaluator = dict(
     type='LyftMetric',
-    ann_file=data_root + 'lyft_infos_val.pkl',
-    metric='bbox',
-    backend_args=backend_args)
-test_evaluator = dict(
-    type='LyftMetric',
-    ann_file=data_root + 'lyft_infos_val.pkl',
+    data_root=data_root,
+    ann_file='lyft_infos_val.pkl',
     metric='bbox',
     backend_args=backend_args)
+test_evaluator = val_evaluator
 
 vis_backends = [dict(type='LocalVisBackend')]
 visualizer = dict(

configs/groupfree3d/README.md

@@ -22,7 +22,7 @@ We implement Group-Free-3D and provide the result and checkpoints on ScanNet dat
 
 |                              Method                              |   Backbone    | Lr schd | Mem (GB) | Inf time (fps) |     AP@0.25     |     AP@0.5      |                                                                                                                                                                                                        Download                                                                                                                                                                                                        |
 | :--------------------------------------------------------------: | :-----------: | :-----: | :------: | :------------: | :-------------: | :-------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
-|    [L6, O256](./groupfree3d_head-L6-O256_4xb8_scannet-seg.py)    |  PointNet++   |   3x    |   6.7    |                | 66.32 (65.67\*) | 47.82 (47.74\*) |           [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347-3499eb55.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347.log.json)           |
+|    [L6, O256](./groupfree3d_head-L6-O256_4xb8_scannet-seg.py)    |  PointNet++   |   3x    |   6.7    |                | 66.17 (65.67\*) | 48.47 (47.74\*) |           [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347-3499eb55.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347.log.json)           |
 |   [L12, O256](./groupfree3d_head-L12-O256_4xb8_scannet-seg.py)   |  PointNet++   |   3x    |   9.4    |                | 66.57 (66.22\*) | 48.21 (48.95\*) |         [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256/groupfree3d_8x4_scannet-3d-18class-L12-O256_20210702_150907-1c5551ad.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256/groupfree3d_8x4_scannet-3d-18class-L12-O256_20210702_150907.log.json)         |
 | [L12, O256](./groupfree3d_w2x-head-L12-O256_4xb8_scannet-seg.py) | PointNet++w2x |   3x    |   13.3   |                | 68.20 (67.30\*) | 51.02 (50.44\*) | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256_20210702_200301-944f0ac0.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256_20210702_200301.log.json) |
 | [L12, O512](./groupfree3d_w2x-head-L12-O512_4xb8_scannet-seg.py) | PointNet++w2x |   3x    |   18.8   |                | 68.22 (68.20\*) | 52.61 (51.31\*) | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512_20210702_220204-187b71c7.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512_20210702_220204.log.json) |

configs/groupfree3d/groupfree3d_head-L6-O256_4xb8_scannet-seg.py

@@ -224,3 +224,4 @@ test_cfg = dict(type='TestLoop')
 
 default_hooks = dict(
     checkpoint=dict(type='CheckpointHook', interval=1, max_keep_ckpts=10))
+randomness = dict(seed=4)

configs/groupfree3d/metafile.yml

@@ -25,8 +25,8 @@ Models:
       - Task: 3D Object Detection
         Dataset: ScanNet
         Metrics:
-          AP@0.25: 66.32
-          AP@0.5: 47.82
+          AP@0.25: 66.17
+          AP@0.5: 48.47
     Weights: https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347-3499eb55.pth
 
   - Name: groupfree3d_head-L12-O256_4xb8_scannet-seg.py

configs/h3dnet/README.md

@@ -21,8 +21,8 @@ We implement H3DNet and provide the result and checkpoints on ScanNet datasets.
 ### ScanNet
 
 |                   Backbone                    | Lr schd | Mem (GB) | Inf time (fps) | AP@0.25 | AP@0.5 |                                                                                                                                                             Download                                                                                                                                                             |
-| :-------------------------------------------: | :-----: | :------: | :------------: | :-----: | :----: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
-| [MultiBackbone](./h3dnet_8xb3_scannet-seg.py) |   3x    |   7.9    |                |  66.07  | 47.68  | [model](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149-414bd304.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149.log.json) |
+| :-------------------------------------------: | :-----: | :------: | :------------: | :-----: | :----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
+| [MultiBackbone](./h3dnet_8xb3_scannet-seg.py) |   3x    |   7.9    |                |  66.07  | 47.68  | [model](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_3x8_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149-414bd304.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149.log.json) |
 
 **Notice**: If your current mmdetection3d version >= 0.6.0, and you are using the checkpoints downloaded from the above links or using checkpoints trained with mmdetection3d version \< 0.6.0, the checkpoints have to be first converted via [tools/model_converters/convert_h3dnet_checkpoints.py](../../tools/model_converters/convert_h3dnet_checkpoints.py):
 

configs/h3dnet/metafile.yml

@@ -26,4 +26,4 @@ Models:
         Metrics:
           AP@0.25: 66.07
           AP@0.5: 47.68
-    Weights: https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149-414bd304.pth
+    Weights: https://download.openmmlab.com/mmdetection3d/v1.0.0_models/h3dnet/h3dnet_3x8_scannet-3d-18class/h3dnet_3x8_scannet-3d-18class_20210824_003149-414bd304.pth

configs/imvotenet/README.md

@@ -28,7 +28,7 @@ We implement ImVoteNet and provide the result and checkpoints on SUNRGBD.
 
 |                       Backbone                       | Lr schd | Mem (GB) | Inf time (fps) | AP@0.25 | AP@0.5 |                                                                                                                                                                                        Download                                                                                                                                                                                        |
 | :--------------------------------------------------: | :-----: | :------: | :------------: | :-----: | :----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
-| [PointNet++](./imvotenet_stage2_8xb16_sunrgbd-3d.py) |   3x    |   9.4    |                |  64.55  |        | [model](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class/imvotenet_stage2_16x8_sunrgbd-3d-10class_20210819_192851-1bcd1b97.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class/imvotenet_stage2_16x8_sunrgbd-3d-10class_20210819_192851.log.json) |
+| [PointNet++](./imvotenet_stage2_8xb16_sunrgbd-3d.py) |   3x    |   9.4    |                |  64.48  |        | [model](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class/imvotenet_stage2_16x8_sunrgbd-3d-10class_20210819_192851-1bcd1b97.pth) \| [log](https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class/imvotenet_stage2_16x8_sunrgbd-3d-10class_20210819_192851.log.json) |
 
 ## Citation
 

configs/imvotenet/imvotenet_faster-rcnn-r50_fpn_4xb2_sunrgbd-3d.py

@@ -25,13 +25,6 @@ train_pipeline = [
 
 test_pipeline = [
     dict(type='LoadImageFromFile', backend_args=backend_args),
-    # online evaluation
-    dict(
-        type='LoadAnnotations3D',
-        with_bbox=True,
-        with_label=True,
-        with_bbox_3d=False,
-        with_label_3d=False),
     dict(type='Resize', scale=(1333, 600), keep_ratio=True),
     dict(
         type='Pack3DDetInputs',
@@ -66,6 +59,7 @@ param_scheduler = [
         gamma=0.1)
 ]
 val_evaluator = dict(type='Indoor2DMetric')
+test_evaluator = val_evaluator
 
 # optimizer
 optim_wrapper = dict(

configs/imvotenet/imvotenet_stage2_8xb16_sunrgbd-3d.py

@@ -219,9 +219,10 @@ val_dataloader = dict(dataset=dict(pipeline=test_pipeline))
 test_dataloader = dict(dataset=dict(pipeline=test_pipeline))
 
 # may also use your own pre-trained image branch
-load_from = 'https://download.openmmlab.com/mmdetection3d/v0.1.0_models/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class_20210323_173222-cad62aeb.pth'  # noqa
+load_from = 'https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class_20210819_225618-62eba6ce.pth'  # noqa
 # Default setting for scaling LR automatically
 #   - `enable` means enable scaling LR automatically
 #       or not by default.
 #   - `base_batch_size` = (8 GPUs) x (16 samples per GPU).
 auto_scale_lr = dict(enable=False, base_batch_size=128)
+randomness = dict(seed=8)

configs/imvotenet/metafile.yml

@@ -39,5 +39,5 @@ Models:
       - Task: 3D Object Detection
         Dataset: SUNRGBD-3D
         Metrics:
-          AP@0.25: 64.55
+          AP@0.25: 64.48
     Weights: https://download.openmmlab.com/mmdetection3d/v1.0.0_models/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class/imvotenet_stage2_16x8_sunrgbd-3d-10class_20210819_192851-1bcd1b97.pth

configs/pv_rcnn/metafile.yml

@@ -26,4 +26,4 @@ Models:
         Dataset: KITTI
         Metrics:
           mAP: 72.28
-    Weights: <https://download.openmmlab.com/mmdetection3d/v1.1.0_models/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class/pv_rcnn_8xb2-80e_kitti-3d-3class_20221117_234428-b384d22f.pth
+    Weights: https://download.openmmlab.com/mmdetection3d/v1.1.0_models/pv_rcnn/pv_rcnn_8xb2-80e_kitti-3d-3class/pv_rcnn_8xb2-80e_kitti-3d-3class_20221117_234428-b384d22f.pth

mmdet3d/evaluation/metrics/lyft_metric.py

@@ -122,7 +122,8 @@ class LyftMetric(BaseMetric):
 
         # load annotations
         self.data_infos = load(
-            self.ann_file, backend_args=self.backend_args)['data_list']
+            osp.join(self.data_root, self.ann_file),
+            backend_args=self.backend_args)['data_list']
         result_dict, tmp_dir = self.format_results(results, classes,
                                                    self.jsonfile_prefix,
                                                    self.csv_savepath)
@@ -360,7 +361,7 @@ def output_to_lyft_box(detection: dict) -> List[LyftBox]:
     Returns:
         List[:obj:`LyftBox`]: List of standard LyftBoxes.
     """
-    bbox3d = detection['bbox_3d']
+    bbox3d = detection['bboxes_3d']
     scores = detection['scores_3d'].numpy()
     labels = detection['labels_3d'].numpy()
 

tools/dataset_converters/lyft_converter.py

@@ -40,10 +40,10 @@ def create_lyft_infos(root_path,
     available_vers = ['v1.01-train', 'v1.01-test']
     assert version in available_vers
     if version == 'v1.01-train':
-        train_scenes = mmcv.list_from_file('data/lyft/train.txt')
-        val_scenes = mmcv.list_from_file('data/lyft/val.txt')
+        train_scenes = mmengine.list_from_file('data/lyft/train.txt')
+        val_scenes = mmengine.list_from_file('data/lyft/val.txt')
     elif version == 'v1.01-test':
-        train_scenes = mmcv.list_from_file('data/lyft/test.txt')
+        train_scenes = mmengine.list_from_file('data/lyft/test.txt')
         val_scenes = []
     else:
         raise ValueError('unknown')

tools/dataset_converters/waymo_converter.py

@@ -6,7 +6,7 @@ r"""Adapted from `Waymo to KITTI converter
 try:
     from waymo_open_dataset import dataset_pb2
 except ImportError:
-    raise ImportError('Please run "pip install waymo-open-dataset-tf-2-5-0" '
+    raise ImportError('Please run "pip install waymo-open-dataset-tf-2-6-0" '
                       '>1.4.5 to install the official devkit first.')
 
 import os