Update waymo dataset, evaluation metrics and related configs for 2.0 (#1663)

* update waymo dataset

* [Fix] Fix all unittests and refactor tests directory and add circle ci in `test-1.x` (#1654)

* add circle ci

* delete github ci

* fix ci

* fix ut

* fix markdown version

* rm

* fix part of uts

* fix comments

* change foler

* refactor test directory

* fix kitti metric ut

* fix all ut
Co-authored-by: VVsssssk <shenkun@pjlab.org.cn>

* add waymo dataset and evaluation metrics

* convert second configs for v2.0

* [Refactor] Unify ceph config (#1677)

* refactor ceph in config

* support metric load ann file from ceph

* add doc string and remove useless code

* [Fix]Fix create data (#1659)

* add circle ci

* delete github ci

* fix ci

* fix ut

* fix markdown version

* rm

* fix part of uts

* fix comments

* change foler

* refactor test directory

* fix kitti metric ut

* fix all ut

* fix creat data
Co-authored-by: ChaimZhu <zhuchenming@pjlab.org.cn>

* [Fix] Fix seg mapping (#1681)

* [Doc]: fix markdown version (#1653)

* [CI] Add circle ci (#1647)

* add circle ci

* delete github ci

* fix ci

* fix ut

* fix markdown version

* rm

* fix seg mapping for scannet

* fix requiremetn

* fix all seg dataet

* resolve commnets
Co-authored-by: VVsssssk <88368822+VVsssssk@users.noreply.github.com>

* [Fix] Fix SSN configs (#1686)

* modify doc string and evaluation file location

* add doc string

* remove path mapping in flieclient args
Co-authored-by: ChaimZhu <zhuchenming@pjlab.org.cn>
Co-authored-by: VVsssssk <shenkun@pjlab.org.cn>
Co-authored-by: VVsssssk <88368822+VVsssssk@users.noreply.github.com>
Co-authored-by: Shilong Zhang <61961338+jshilong@users.noreply.github.com>

mmdet3d/evaluation/__init__.py

@@ -9,7 +9,7 @@ from .functional import (aggregate_predictions, average_precision,
                          load_lyft_predictions, lyft_eval, per_class_iou,
                          rename_gt, seg_eval)
 from .metrics import (IndoorMetric, InstanceSegMetric, KittiMetric, LyftMetric,
-                      NuScenesMetric, SegMetric)
+                      NuScenesMetric, SegMetric, WaymoMetric)
 
 __all__ = [
     'kitti_eval_coco_style', 'kitti_eval', 'indoor_eval', 'lyft_eval',
@@ -19,5 +19,5 @@ __all__ = [
     'get_classwise_aps', 'get_single_class_aps', 'fast_hist', 'per_class_iou',
     'get_acc', 'get_acc_cls', 'seg_eval', 'KittiMetric', 'NuScenesMetric',
     'IndoorMetric', 'LyftMetric', 'SegMetric', 'InstanceSegMetric',
-    'eval_class', 'do_eval'
+    'WaymoMetric', 'eval_class', 'do_eval'
 ]

mmdet3d/evaluation/metrics/__init__.py

@@ -5,8 +5,9 @@ from .kitti_metric import KittiMetric  # noqa: F401,F403
 from .lyft_metric import LyftMetric  # noqa: F401,F403
 from .nuscenes_metric import NuScenesMetric  # noqa: F401,F403
 from .seg_metric import SegMetric  # noqa: F401,F403
+from .waymo_metric import WaymoMetric  # noqa: F401,F403
 
 __all_ = [
     'KittiMetric', 'NuScenesMetric', 'IndoorMetric', 'LyftMetric', 'SegMetric',
-    'InstanceSegMetric'
+    'InstanceSegMetric', 'WaymoMetric'
 ]

mmdet3d/evaluation/metrics/instance_seg_metric.py

@@ -69,7 +69,7 @@ class InstanceSegMetric(BaseMetric):
         logger: MMLogger = MMLogger.get_current_instance()
 
         self.classes = self.dataset_meta['CLASSES']
-        self.valid_class_ids = self.dataset_meta['VALID_CLASS_IDS']
+        self.valid_class_ids = self.dataset_meta['seg_valid_class_ids']
 
         gt_semantic_masks = []
         gt_instance_masks = []
@@ -80,9 +80,10 @@ class InstanceSegMetric(BaseMetric):
         for eval_ann, sinlge_pred_results in results:
             gt_semantic_masks.append(eval_ann['pts_semantic_mask'])
             gt_instance_masks.append(eval_ann['pts_instance_mask'])
-            pred_instance_masks.append(sinlge_pred_results['pts_intance_mask'])
-            pred_instance_labels.append(sinlge_pred_results['instance_label'])
-            pred_instance_scores.append(sinlge_pred_results['instance_score'])
+            pred_instance_masks.append(
+                sinlge_pred_results['pts_instance_mask'])
+            pred_instance_labels.append(sinlge_pred_results['instance_labels'])
+            pred_instance_scores.append(sinlge_pred_results['instance_scores'])
 
         ret_dict = instance_seg_eval(
             gt_semantic_masks,

mmdet3d/evaluation/metrics/kitti_metric.py

@@ -7,6 +7,7 @@ import mmcv
 import numpy as np
 import torch
 from mmcv.utils import print_log
+from mmengine import load
 from mmengine.evaluator import BaseMetric
 from mmengine.logging import MMLogger
 
@@ -34,6 +35,8 @@ class KittiMetric(BaseMetric):
         pklfile_prefix (str, optional): The prefix of pkl files, including
             the file path and the prefix of filename, e.g., "a/b/prefix".
             If not specified, a temp file will be created. Default: None.
+        default_cam_key (str, optional): The default camera for lidar to
+            camear conversion. By default, KITTI: CAM2, Waymo: CAM_FRONT
         submission_prefix (str, optional): The prefix of submission data.
             If not specified, the submission data will not be generated.
             Default: None.
@@ -49,8 +52,10 @@ class KittiMetric(BaseMetric):
                  pcd_limit_range: List[float] = [0, -40, -3, 70.4, 40, 0.0],
                  prefix: Optional[str] = None,
                  pklfile_prefix: str = None,
+                 default_cam_key: str = 'CAM2',
                  submission_prefix: str = None,
-                 collect_device: str = 'cpu'):
+                 collect_device: str = 'cpu',
+                 file_client_args: dict = dict(backend='disk')):
         self.default_prefix = 'Kitti metric'
         super(KittiMetric, self).__init__(
             collect_device=collect_device, prefix=prefix)
@@ -59,6 +64,7 @@ class KittiMetric(BaseMetric):
         self.pklfile_prefix = pklfile_prefix
         self.submission_prefix = submission_prefix
         self.pred_box_type_3d = pred_box_type_3d
+        self.file_client_args = file_client_args
 
         allowed_metrics = ['bbox', 'img_bbox', 'mAP']
         self.metrics = metric if isinstance(metric, list) else [metric]
@@ -131,18 +137,6 @@ class KittiMetric(BaseMetric):
             data_annos[i]['kitti_annos'] = kitti_annos
         return data_annos
 
-    def load_annotations(self, ann_file: str) -> list:
-        """Load annotations from ann_file.
-
-        Args:
-            ann_file (str): Path of the annotation file.
-
-        Returns:
-            list[dict]: List of annotations.
-        """
-        # loading data from a pkl file
-        return mmcv.load(ann_file, file_format='pkl')
-
     def process(self, data_batch: Sequence[dict],
                 predictions: Sequence[dict]) -> None:
         """Process one batch of data samples and predictions.
@@ -183,7 +177,8 @@ class KittiMetric(BaseMetric):
         self.classes = self.dataset_meta['CLASSES']
 
         # load annotations
-        pkl_annos = self.load_annotations(self.ann_file)['data_list']
+        pkl_annos = load(
+            self.ann_file, file_client_args=self.file_client_args)['data_list']
         self.data_infos = self.convert_annos_to_kitti_annos(pkl_annos)
         result_dict, tmp_dir = self.format_results(
             results,
@@ -340,8 +335,8 @@ class KittiMetric(BaseMetric):
             info = self.data_infos[sample_idx]
             # Here default used 'CAM2' to compute metric. If you want to
             # use another camera, please modify it.
-            image_shape = (info['images']['CAM2']['height'],
-                           info['images']['CAM2']['width'])
+            image_shape = (info['images'][self.default_cam_key]['height'],
+                           info['images'][self.default_cam_key]['width'])
             box_dict = self.convert_valid_bboxes(pred_dicts, info)
             anno = {
                 'name': [],
@@ -590,11 +585,13 @@ class KittiMetric(BaseMetric):
                 sample_idx=sample_idx)
         # Here default used 'CAM2' to compute metric. If you want to
         # use another camera, please modify it.
-        lidar2cam = np.array(info['images']['CAM2']['lidar2cam']).astype(
+        lidar2cam = np.array(
+            info['images'][self.default_cam_key]['lidar2cam']).astype(
+                np.float32)
+        P2 = np.array(info['images'][self.default_cam_key]['cam2img']).astype(
             np.float32)
-        P2 = np.array(info['images']['CAM2']['cam2img']).astype(np.float32)
-        img_shape = (info['images']['CAM2']['height'],
-                     info['images']['CAM2']['width'])
+        img_shape = (info['images'][self.default_cam_key]['height'],
+                     info['images'][self.default_cam_key]['width'])
         P2 = box_preds.tensor.new_tensor(P2)
 
         if isinstance(box_preds, LiDARInstance3DBoxes):

mmdet3d/evaluation/metrics/lyft_metric.py

@@ -10,6 +10,7 @@ import numpy as np
 import pandas as pd
 from lyft_dataset_sdk.lyftdataset import LyftDataset as Lyft
 from lyft_dataset_sdk.utils.data_classes import Box as LyftBox
+from mmengine import load
 from mmengine.evaluator import BaseMetric
 from mmengine.logging import MMLogger
 from pyquaternion import Quaternion
@@ -45,18 +46,21 @@ class LyftMetric(BaseMetric):
             'gpu'. Defaults to 'cpu'.
     """
 
-    def __init__(self,
-                 data_root: str,
-                 ann_file: str,
-                 metric: Union[str, List[str]] = 'bbox',
-                 modality=dict(
-                     use_camera=False,
-                     use_lidar=True,
-                 ),
-                 prefix: Optional[str] = None,
-                 jsonfile_prefix: str = None,
-                 csv_savepath: str = None,
-                 collect_device: str = 'cpu') -> None:
+    def __init__(
+        self,
+        data_root: str,
+        ann_file: str,
+        metric: Union[str, List[str]] = 'bbox',
+        modality=dict(
+            use_camera=False,
+            use_lidar=True,
+        ),
+        prefix: Optional[str] = None,
+        jsonfile_prefix: str = None,
+        csv_savepath: str = None,
+        collect_device: str = 'cpu',
+        file_client_args: dict = dict(backend='disk')
+    ) -> None:
         self.default_prefix = 'Lyft metric'
         super(LyftMetric, self).__init__(
             collect_device=collect_device, prefix=prefix)
@@ -64,21 +68,10 @@ class LyftMetric(BaseMetric):
         self.data_root = data_root
         self.modality = modality
         self.jsonfile_prefix = jsonfile_prefix
+        self.file_client_args = file_client_args
         self.csv_savepath = csv_savepath
         self.metrics = metric if isinstance(metric, list) else [metric]
 
-    def load_annotations(self, ann_file: str) -> list:
-        """Load annotations from ann_file.
-
-        Args:
-            ann_file (str): Path of the annotation file.
-
-        Returns:
-            list[dict]: List of annotations.
-        """
-        # loading data from a pkl file
-        return mmcv.load(ann_file, file_format='pkl')
-
     def process(self, data_batch: Sequence[dict],
                 predictions: Sequence[dict]) -> None:
         """Process one batch of data samples and predictions.
@@ -122,7 +115,8 @@ class LyftMetric(BaseMetric):
         self.version = self.dataset_meta['version']
         # load annotations
 
-        self.data_infos = self.load_annotations(self.ann_file)['data_list']
+        self.data_infos = load(
+            self.ann_file, file_client_args=self.file_client_args)['data_list']
         result_dict, tmp_dir = self.format_results(results, classes,
                                                    self.jsonfile_prefix)
 

mmdet3d/evaluation/metrics/nuscenes_metric.py

@@ -8,6 +8,7 @@ import mmcv
 import numpy as np
 import pyquaternion
 import torch
+from mmengine import load
 from mmengine.evaluator import BaseMetric
 from mmengine.logging import MMLogger
 from nuscenes.eval.detection.config import config_factory
@@ -81,15 +82,18 @@ class NuScenesMetric(BaseMetric):
         'attr_err': 'mAAE'
     }
 
-    def __init__(self,
-                 data_root: str,
-                 ann_file: str,
-                 metric: Union[str, List[str]] = 'bbox',
-                 modality: Dict = dict(use_camera=False, use_lidar=True),
-                 prefix: Optional[str] = None,
-                 jsonfile_prefix: Optional[str] = None,
-                 eval_version: str = 'detection_cvpr_2019',
-                 collect_device: str = 'cpu') -> None:
+    def __init__(
+        self,
+        data_root: str,
+        ann_file: str,
+        metric: Union[str, List[str]] = 'bbox',
+        modality: Dict = dict(use_camera=False, use_lidar=True),
+        prefix: Optional[str] = None,
+        jsonfile_prefix: Optional[str] = None,
+        eval_version: str = 'detection_cvpr_2019',
+        collect_device: str = 'cpu',
+        file_client_args: dict = dict(backend='disk')
+    ) -> None:
         self.default_prefix = 'NuScenes metric'
         super(NuScenesMetric, self).__init__(
             collect_device=collect_device, prefix=prefix)
@@ -102,24 +106,13 @@ class NuScenesMetric(BaseMetric):
         self.data_root = data_root
         self.modality = modality
         self.jsonfile_prefix = jsonfile_prefix
+        self.file_client_args = file_client_args
 
         self.metrics = metric if isinstance(metric, list) else [metric]
 
         self.eval_version = eval_version
         self.eval_detection_configs = config_factory(self.eval_version)
 
-    def load_annotations(self, ann_file: str) -> list:
-        """Load annotations from ann_file.
-
-        Args:
-            ann_file (str): Path of the annotation file.
-
-        Returns:
-            list[dict]: List of annotations.
-        """
-        # loading data from a pkl file
-        return mmcv.load(ann_file, file_format='pkl')
-
     def process(self, data_batch: Sequence[dict],
                 predictions: Sequence[dict]) -> None:
         """Process one batch of data samples and predictions.
@@ -162,7 +155,8 @@ class NuScenesMetric(BaseMetric):
         classes = self.dataset_meta['CLASSES']
         self.version = self.dataset_meta['version']
         # load annotations
-        self.data_infos = self.load_annotations(self.ann_file)['data_list']
+        self.data_infos = load(
+            self.ann_file, file_client_args=self.file_client_args)['data_list']
         result_dict, tmp_dir = self.format_results(results, classes,
                                                    self.jsonfile_prefix)
 

mmdet3d/evaluation/metrics/waymo_metric.py

+# Copyright (c) OpenMMLab. All rights reserved.
+import tempfile
+from os import path as osp
+from typing import Dict, List, Optional, Union
+
+import mmcv
+import numpy as np
+import torch
+from mmcv.utils import print_log
+from mmengine.logging import MMLogger
+
+from mmdet3d.models.layers import box3d_multiclass_nms
+from mmdet3d.registry import METRICS
+from mmdet3d.structures import (Box3DMode, LiDARInstance3DBoxes, bbox3d2result,
+                                xywhr2xyxyr)
+from .kitti_metric import KittiMetric
+
+
+@METRICS.register_module()
+class WaymoMetric(KittiMetric):
+    """Waymo evaluation metric.
+
+    Args:
+        ann_file (str): The path of the annotation file in kitti format.
+        waymo_bin_file (str): The path of the annotation file in waymo format.
+        data_root (str): Path of dataset root.
+                         Used for storing waymo evaluation programs.
+        split (str): The split of the evaluation set.
+        metric (str | list[str]): Metrics to be evaluated.
+            Default to 'bbox'.
+        pcd_limit_range (list): The range of point cloud used to
+            filter invalid predicted boxes.
+            Default to [0, -40, -3, 70.4, 40, 0.0].
+        prefix (str, optional): The prefix that will be added in the metric
+            names to disambiguate homonymous metrics of different evaluators.
+            If prefix is not provided in the argument, self.default_prefix
+            will be used instead. Defaults to None.
+        pklfile_prefix (str, optional): The prefix of pkl files, including
+            the file path and the prefix of filename, e.g., "a/b/prefix".
+            If not specified, a temp file will be created. Default: None.
+        submission_prefix (str, optional): The prefix of submission data.
+            If not specified, the submission data will not be generated.
+            Default: None.
+        task: (str, optional): task for 3D detection, if cam, would filter
+            the points that outside the image.
+        default_cam_key (str, optional): The default camera for lidar to
+            camear conversion. By default, KITTI: CAM2, Waymo: CAM_FRONT
+        use_pred_sample_idx (bool, optional): In formating results, use the
+            sample index from the prediction or from the load annoataitons.
+            By default, KITTI: True, Waymo: False, Waymo has a conversion
+            process, which needs to use the sample id from load annotation.
+        collect_device (str): Device name used for collecting results
+            from different ranks during distributed training. Must be 'cpu' or
+            'gpu'. Defaults to 'cpu'.
+        file_client_args (dict): file client for reading gt in waymo format.
+    """
+
+    def __init__(self,
+                 ann_file: str,
+                 waymo_bin_file: str,
+                 data_root: str,
+                 split: str = 'training',
+                 metric: Union[str, List[str]] = 'bbox',
+                 pcd_limit_range: List[float] = [-85, -85, -5, 85, 85, 5],
+                 prefix: Optional[str] = None,
+                 pklfile_prefix: str = None,
+                 submission_prefix: str = None,
+                 task='lidar',
+                 default_cam_key: str = 'CAM_FRONT',
+                 use_pred_sample_idx: bool = False,
+                 collect_device: str = 'cpu',
+                 file_client_args: dict = dict(backend='disk')):
+
+        self.waymo_bin_file = waymo_bin_file
+        self.data_root = data_root
+        self.file_client_args = file_client_args
+        self.split = split
+        self.task = task
+        self.use_pred_sample_idx = use_pred_sample_idx
+        super().__init__(
+            ann_file=ann_file,
+            metric=metric,
+            pcd_limit_range=pcd_limit_range,
+            prefix=prefix,
+            pklfile_prefix=pklfile_prefix,
+            submission_prefix=submission_prefix,
+            default_cam_key=default_cam_key,
+            collect_device=collect_device)
+        self.default_prefix = 'Waymo metric'
+
+    def compute_metrics(self, results: list) -> Dict[str, float]:
+        """Compute the metrics from processed results.
+
+        Args:
+            results (list): The processed results of each batch.
+
+        Returns:
+            Dict[str, float]: The computed metrics. The keys are the names of
+            the metrics, and the values are corresponding results.
+        """
+        logger: MMLogger = MMLogger.get_current_instance()
+        self.classes = self.dataset_meta['CLASSES']
+
+        # load annotations
+        self.data_infos = self.load_annotations(self.ann_file)['data_list']
+        # different from kitti, waymo do not need to convert the ann file
+
+        if self.pklfile_prefix is None:
+            eval_tmp_dir = tempfile.TemporaryDirectory()
+            pklfile_prefix = osp.join(eval_tmp_dir.name, 'results')
+        else:
+            eval_tmp_dir = None
+            pklfile_prefix = self.pklfile_prefix
+
+        # load annotations
+
+        result_dict, tmp_dir = self.format_results(
+            results,
+            pklfile_prefix=pklfile_prefix,
+            submission_prefix=self.submission_prefix,
+            classes=self.classes)
+
+        import subprocess
+        eval_str = 'mmdet3d/evaluation/functional/waymo_utils/' + \
+            f'compute_detection_metrics_main {pklfile_prefix}.bin ' + \
+            f'{self.waymo_bin_file}'
+        print(eval_str)
+        ret_bytes = subprocess.check_output(eval_str, shell=True)
+        ret_texts = ret_bytes.decode('utf-8')
+        print_log(ret_texts, logger=logger)
+
+        ap_dict = {
+            'Vehicle/L1 mAP': 0,
+            'Vehicle/L1 mAPH': 0,
+            'Vehicle/L2 mAP': 0,
+            'Vehicle/L2 mAPH': 0,
+            'Pedestrian/L1 mAP': 0,
+            'Pedestrian/L1 mAPH': 0,
+            'Pedestrian/L2 mAP': 0,
+            'Pedestrian/L2 mAPH': 0,
+            'Sign/L1 mAP': 0,
+            'Sign/L1 mAPH': 0,
+            'Sign/L2 mAP': 0,
+            'Sign/L2 mAPH': 0,
+            'Cyclist/L1 mAP': 0,
+            'Cyclist/L1 mAPH': 0,
+            'Cyclist/L2 mAP': 0,
+            'Cyclist/L2 mAPH': 0,
+            'Overall/L1 mAP': 0,
+            'Overall/L1 mAPH': 0,
+            'Overall/L2 mAP': 0,
+            'Overall/L2 mAPH': 0
+        }
+        mAP_splits = ret_texts.split('mAP ')
+        mAPH_splits = ret_texts.split('mAPH ')
+        mAP_splits = ret_texts.split('mAP ')
+        mAPH_splits = ret_texts.split('mAPH ')
+        for idx, key in enumerate(ap_dict.keys()):
+            split_idx = int(idx / 2) + 1
+            if idx % 2 == 0:  # mAP
+                ap_dict[key] = float(mAP_splits[split_idx].split(']')[0])
+            else:  # mAPH
+                ap_dict[key] = float(mAPH_splits[split_idx].split(']')[0])
+        ap_dict['Overall/L1 mAP'] = \
+            (ap_dict['Vehicle/L1 mAP'] + ap_dict['Pedestrian/L1 mAP'] +
+                ap_dict['Cyclist/L1 mAP']) / 3
+        ap_dict['Overall/L1 mAPH'] = \
+            (ap_dict['Vehicle/L1 mAPH'] + ap_dict['Pedestrian/L1 mAPH'] +
+                ap_dict['Cyclist/L1 mAPH']) / 3
+        ap_dict['Overall/L2 mAP'] = \
+            (ap_dict['Vehicle/L2 mAP'] + ap_dict['Pedestrian/L2 mAP'] +
+                ap_dict['Cyclist/L2 mAP']) / 3
+        ap_dict['Overall/L2 mAPH'] = \
+            (ap_dict['Vehicle/L2 mAPH'] + ap_dict['Pedestrian/L2 mAPH'] +
+                ap_dict['Cyclist/L2 mAPH']) / 3
+        if eval_tmp_dir is not None:
+            eval_tmp_dir.cleanup()
+
+        if tmp_dir is not None:
+            tmp_dir.cleanup()
+
+        return ap_dict
+
+    def format_results(self,
+                       results: List[dict],
+                       pklfile_prefix: str = None,
+                       submission_prefix: str = None,
+                       classes: List[str] = None):
+        """Format the results to pkl file.
+
+        Args:
+            results (list[dict]): Testing results of the
+                dataset.
+            pklfile_prefix (str, optional): The prefix of pkl files. It
+                includes the file path and the prefix of filename, e.g.,
+                "a/b/prefix". If not specified, a temp file will be created.
+                Default: None.
+            submission_prefix (str, optional): The prefix of submitted files.
+                It includes the file path and the prefix of filename, e.g.,
+                "a/b/prefix". If not specified, a temp file will be created.
+                Default: None.
+            classes (list[String], optional): A list of class name. Defaults
+                to None.
+
+        Returns:
+            tuple: (result_dict, tmp_dir), result_dict is a dict containing
+                the formatted result, tmp_dir is the temporal directory created
+                for saving json files when jsonfile_prefix is not specified.
+        """
+        result_files, tmp_dir = super().format_results(results, pklfile_prefix,
+                                                       submission_prefix,
+                                                       classes)
+
+        waymo_root = self.data_root
+        if self.split == 'training':
+            waymo_tfrecords_dir = osp.join(waymo_root, 'validation')
+            prefix = '1'
+        elif self.split == 'testing':
+            waymo_tfrecords_dir = osp.join(waymo_root, 'testing')
+            prefix = '2'
+        else:
+            raise ValueError('Not supported split value.')
+        waymo_save_tmp_dir = tempfile.TemporaryDirectory()
+        waymo_results_save_dir = waymo_save_tmp_dir.name
+        waymo_results_final_path = f'{pklfile_prefix}.bin'
+        from ..core.evaluation.waymo_utils.prediction_kitti_to_waymo import \
+            KITTI2Waymo
+        converter = KITTI2Waymo(
+            result_files['pred_instances_3d'],
+            waymo_tfrecords_dir,
+            waymo_results_save_dir,
+            waymo_results_final_path,
+            prefix,
+            file_client_args=self.file_client_args)
+        converter.convert()
+        waymo_save_tmp_dir.cleanup()
+        return result_files, waymo_save_tmp_dir
+
+    def merge_multi_view_boxes(self, box_dict_per_frame: List[dict],
+                               cam0_info: dict):
+        """Merge bounding boxes predicted from multi-view images.
+        Args:
+            box_dict_per_frame (list[dict]): The results of prediction
+                for each camera.
+            cam2_info (dict): store the sample id for the given frame.
+
+        Returns:
+            merged_box_dict (dict), store the merge results
+        """
+        box_dict = dict()
+        # convert list[dict] to dict[list]
+        for key in box_dict_per_frame[0].keys():
+            box_dict[key] = list()
+            for cam_idx in range(self.num_cams):
+                box_dict[key].append(box_dict_per_frame[cam_idx][key])
+        # merge each elements
+        box_dict['sample_id'] = cam0_info['image_id']
+        for key in ['bbox', 'box3d_lidar', 'scores', 'label_preds']:
+            box_dict[key] = np.concatenate(box_dict[key])
+
+        # apply nms to box3d_lidar (box3d_camera are in different systems)
+        # TODO: move this global setting into config
+        nms_cfg = dict(
+            use_rotate_nms=True,
+            nms_across_levels=False,
+            nms_pre=500,
+            nms_thr=0.05,
+            score_thr=0.001,
+            min_bbox_size=0,
+            max_per_frame=100)
+        from mmcv import Config
+        nms_cfg = Config(nms_cfg)
+        lidar_boxes3d = LiDARInstance3DBoxes(
+            torch.from_numpy(box_dict['box3d_lidar']).cuda())
+        scores = torch.from_numpy(box_dict['scores']).cuda()
+        labels = torch.from_numpy(box_dict['label_preds']).long().cuda()
+        nms_scores = scores.new_zeros(scores.shape[0], len(self.CLASSES) + 1)
+        indices = labels.new_tensor(list(range(scores.shape[0])))
+        nms_scores[indices, labels] = scores
+        lidar_boxes3d_for_nms = xywhr2xyxyr(lidar_boxes3d.bev)
+        boxes3d = lidar_boxes3d.tensor
+        # generate attr scores from attr labels
+        boxes3d, scores, labels = box3d_multiclass_nms(
+            boxes3d, lidar_boxes3d_for_nms, nms_scores, nms_cfg.score_thr,
+            nms_cfg.max_per_frame, nms_cfg)
+        lidar_boxes3d = LiDARInstance3DBoxes(boxes3d)
+        det = bbox3d2result(lidar_boxes3d, scores, labels)
+        box_preds_lidar = det['boxes_3d']
+        scores = det['scores_3d']
+        labels = det['labels_3d']
+        # box_preds_camera is in the cam0 system
+        rect = cam0_info['calib']['R0_rect'].astype(np.float32)
+        Trv2c = cam0_info['calib']['Tr_velo_to_cam'].astype(np.float32)
+        box_preds_camera = box_preds_lidar.convert_to(
+            Box3DMode.CAM, rect @ Trv2c, correct_yaw=True)
+        # Note: bbox is meaningless in final evaluation, set to 0
+        merged_box_dict = dict(
+            bbox=np.zeros([box_preds_lidar.tensor.shape[0], 4]),
+            box3d_camera=box_preds_camera.tensor.numpy(),
+            box3d_lidar=box_preds_lidar.tensor.numpy(),
+            scores=scores.numpy(),
+            label_preds=labels.numpy(),
+            sample_idx=box_dict['sample_idx'],
+        )
+        return merged_box_dict
+
+    def bbox2result_kitti(self,
+                          net_outputs: list,
+                          sample_id_list: list,
+                          class_names: list,
+                          pklfile_prefix: str = None,
+                          submission_prefix: str = None):
+        """Convert 3D detection results to kitti format for evaluation and test
+        submission.
+
+        Args:
+            net_outputs (list[dict]): List of array storing the
+                inferenced bounding boxes and scores.
+            sample_id_list (list[int]): List of input sample id.
+            class_names (list[String]): A list of class names.
+            pklfile_prefix (str, optional): The prefix of pkl file.
+                Defaults to None.
+            submission_prefix (str, optional): The prefix of submission file.
+                Defaults to None.
+
+        Returns:
+            list[dict]: A list of dictionaries with the kitti format.
+        """
+        assert len(net_outputs) == len(self.data_infos), \
+            'invalid list length of network outputs'
+        if submission_prefix is not None:
+            mmcv.mkdir_or_exist(submission_prefix)
+
+        det_annos = []
+        print('\nConverting prediction to KITTI format')
+        for idx, pred_dicts in enumerate(
+                mmcv.track_iter_progress(net_outputs)):
+            annos = []
+            sample_idx = sample_id_list[idx]
+            info = self.data_infos[sample_idx]
+            # Here default used 'CAM2' to compute metric. If you want to
+            # use another camera, please modify it.
+            image_shape = (info['images'][self.default_cam_key]['height'],
+                           info['images'][self.default_cam_key]['width'])
+
+            if self.task == 'mono3d':
+                if idx % self.num_cams == 0:
+                    box_dict_per_frame = []
+                    cam0_idx = idx
+            box_dict = self.convert_valid_bboxes(pred_dicts, info)
+
+            if self.task == 'mono3d':
+                box_dict_per_frame.append(box_dict)
+                if (idx + 1) % self.num_cams != 0:
+                    continue
+                box_dict = self.merge_multi_view_boxes(
+                    box_dict_per_frame, self.data_infos[cam0_idx])
+            anno = {
+                'name': [],
+                'truncated': [],
+                'occluded': [],
+                'alpha': [],
+                'bbox': [],
+                'dimensions': [],
+                'location': [],
+                'rotation_y': [],
+                'score': []
+            }
+            if len(box_dict['bbox']) > 0:
+                box_2d_preds = box_dict['bbox']
+                box_preds = box_dict['box3d_camera']
+                scores = box_dict['scores']
+                box_preds_lidar = box_dict['box3d_lidar']
+                label_preds = box_dict['label_preds']
+
+                for box, box_lidar, bbox, score, label in zip(
+                        box_preds, box_preds_lidar, box_2d_preds, scores,
+                        label_preds):
+                    bbox[2:] = np.minimum(bbox[2:], image_shape[::-1])
+                    bbox[:2] = np.maximum(bbox[:2], [0, 0])
+                    anno['name'].append(class_names[int(label)])
+                    anno['truncated'].append(0.0)
+                    anno['occluded'].append(0)
+                    anno['alpha'].append(
+                        -np.arctan2(-box_lidar[1], box_lidar[0]) + box[6])
+                    anno['bbox'].append(bbox)
+                    anno['dimensions'].append(box[3:6])
+                    anno['location'].append(box[:3])
+                    anno['rotation_y'].append(box[6])
+                    anno['score'].append(score)
+
+                anno = {k: np.stack(v) for k, v in anno.items()}
+                annos.append(anno)
+            else:
+                anno = {
+                    'name': np.array([]),
+                    'truncated': np.array([]),
+                    'occluded': np.array([]),
+                    'alpha': np.array([]),
+                    'bbox': np.zeros([0, 4]),
+                    'dimensions': np.zeros([0, 3]),
+                    'location': np.zeros([0, 3]),
+                    'rotation_y': np.array([]),
+                    'score': np.array([]),
+                }
+                annos.append(anno)
+
+            if submission_prefix is not None:
+                curr_file = f'{submission_prefix}/{sample_idx:06d}.txt'
+                with open(curr_file, 'w') as f:
+                    bbox = anno['bbox']
+                    loc = anno['location']
+                    dims = anno['dimensions']  # lhw -> hwl
+
+                    for idx in range(len(bbox)):
+                        print(
+                            '{} -1 -1 {:.4f} {:.4f} {:.4f} {:.4f} '
+                            '{:.4f} {:.4f} {:.4f} '
+                            '{:.4f} {:.4f} {:.4f} {:.4f} {:.4f} {:.4f}'.format(
+                                anno['name'][idx], anno['alpha'][idx],
+                                bbox[idx][0], bbox[idx][1], bbox[idx][2],
+                                bbox[idx][3], dims[idx][1], dims[idx][2],
+                                dims[idx][0], loc[idx][0], loc[idx][1],
+                                loc[idx][2], anno['rotation_y'][idx],
+                                anno['score'][idx]),
+                            file=f)
+            if self.use_pred_sample_idx:
+                save_sample_idx = sample_idx
+            else:
+                # use the sample idx in the info file
+                # In waymo validation sample_idx in prediction is 000xxx
+                # but in info file it is 1000xxx
+                save_sample_idx = box_dict['sample_idx']
+            annos[-1]['sample_id'] = np.array(
+                [save_sample_idx] * len(annos[-1]['score']), dtype=np.int64)
+
+            det_annos += annos
+
+        if pklfile_prefix is not None:
+            if not pklfile_prefix.endswith(('.pkl', '.pickle')):
+                out = f'{pklfile_prefix}.pkl'
+            else:
+                out = pklfile_prefix
+            mmcv.dump(det_annos, out)
+            print(f'Result is saved to {out}.')
+
+        return det_annos

mmdet3d/models/decode_heads/decode_head.py

@@ -107,8 +107,8 @@ class Base3DDecodeHead(BaseModule, metaclass=ABCMeta):
         output = self.conv_seg(feat)
         return output
 
-    def loss(self, inputs: List[Tensor], batch_data_samples: SampleList,
-             train_cfg: ConfigType) -> dict:
+    def loss(self, inputs: List[Tensor],
+             batch_data_samples: SampleList) -> dict:
         """Forward function for training.
 
         Args:

mmdet3d/structures/point_data.py

@@ -20,7 +20,10 @@ class PointData(BaseDataElement):
     - They are all one dimension.
     - They should have the same length.
 
-    Notice: ``PointData`` behaves like `InstanceData`.
+    `PointData` is used to save point-level semantic and instance mask,
+    it also can save `instances_labels` and `instances_scores` temporarily.
+    In the future, we would consider to move the instance-level info into
+    `gt_instances_3d` and `pred_instances_3d`.
 
     Examples:
         >>> metainfo = dict(
@@ -59,14 +62,6 @@ class PointData(BaseDataElement):
         else:
             assert isinstance(value,
                               Sized), 'value must contain `_len__` attribute'
-
-            if len(self) > 0:
-                assert len(value) == len(self), f'the length of ' \
-                                                f'values {len(value)} is ' \
-                                                f'not consistent with' \
-                                                f' the length of this ' \
-                                                f':obj:`PointData` ' \
-                                                f'{len(self)} '
             super().__setattr__(name, value)
 
     __setitem__ = __setattr__

mmdet3d/utils/__init__.py

@@ -3,6 +3,7 @@ from .array_converter import ArrayConverter, array_converter
 from .collect_env import collect_env
 from .compat_cfg import compat_cfg
 from .logger import get_root_logger
+from .misc import replace_ceph_backend
 from .setup_env import register_all_modules, setup_multi_processes
 from .typing import (ConfigType, InstanceList, MultiConfig, OptConfigType,
                      OptInstanceList, OptMultiConfig, OptSamplingResultList)
@@ -12,5 +13,5 @@ __all__ = [
     'register_all_modules', 'find_latest_checkpoint', 'array_converter',
     'ArrayConverter', 'ConfigType', 'OptConfigType', 'MultiConfig',
     'OptMultiConfig', 'InstanceList', 'OptInstanceList',
-    'OptSamplingResultList'
+    'OptSamplingResultList', 'replace_ceph_backend'
 ]

mmdet3d/utils/collect_env.py

@@ -4,14 +4,12 @@ from mmcv.utils import get_git_hash
 
 import mmdet
 import mmdet3d
-import mmseg
 
 
 def collect_env():
     """Collect the information of the running environments."""
     env_info = collect_base_env()
     env_info['MMDetection'] = mmdet.__version__
-    env_info['MMSegmentation'] = mmseg.__version__
     env_info['MMDetection3D'] = mmdet3d.__version__ + '+' + get_git_hash()[:7]
     from mmdet3d.models.layers.spconv import IS_SPCONV2_AVAILABLE
     env_info['spconv2.0'] = IS_SPCONV2_AVAILABLE

mmdet3d/utils/misc.py

@@ -37,3 +37,78 @@ def find_latest_checkpoint(path, suffix='pth'):
             latest = count
             latest_path = checkpoint
     return latest_path
+
+
+def replace_ceph_backend(cfg):
+    cfg_pretty_text = cfg.pretty_text
+
+    replace_strs = r'''file_client_args = dict(
+                    backend='petrel',
+                    path_mapping=dict({
+                        '.data/INPLACEHOLD/':
+                        's3://openmmlab/datasets/detection3d/INPLACEHOLD/',
+                        'data/INPLACEHOLD/':
+                        's3://openmmlab/datasets/detection3d/INPLACEHOLD/'
+                    }))
+                '''
+
+    if 'nuscenes' in cfg_pretty_text:
+        replace_strs = replace_strs.replace('INPLACEHOLD', 'nuscenes')
+    elif 'lyft' in cfg_pretty_text:
+        replace_strs = replace_strs.replace('INPLACEHOLD', 'lyft')
+    elif 'kitti' in cfg_pretty_text:
+        replace_strs = replace_strs.replace('INPLACEHOLD', 'kitti')
+    elif 'waymo' in cfg_pretty_text:
+        replace_strs = replace_strs.replace('INPLACEHOLD', 'waymo')
+    elif 'scannet' in cfg_pretty_text:
+        replace_strs = replace_strs.replace('INPLACEHOLD', 'scannet_processed')
+    elif 's3dis' in cfg_pretty_text:
+        replace_strs = replace_strs.replace('INPLACEHOLD', 's3dis_processed')
+    elif 'sunrgbd' in cfg_pretty_text:
+        replace_strs = replace_strs.replace('INPLACEHOLD', 'sunrgbd')
+    elif 'semantickitti' in cfg_pretty_text:
+        replace_strs = replace_strs.replace('INPLACEHOLD', 'semantickitti')
+    elif 'nuimages' in cfg_pretty_text:
+        replace_strs = replace_strs.replace('INPLACEHOLD', 'nuimages')
+    else:
+        NotImplemented('Does not support global replacement')
+
+    replace_strs = replace_strs.replace(' ', '').replace('\n', '')
+
+    # use data info file from ceph
+    # cfg_pretty_text = cfg_pretty_text.replace(
+    #   'ann_file', replace_strs + ', ann_file')
+
+    # replace LoadImageFromFile
+    replace_strs = replace_strs.replace(' ', '').replace('\n', '')
+    cfg_pretty_text = cfg_pretty_text.replace(
+        'LoadImageFromFile\'', 'LoadImageFromFile\',' + replace_strs)
+
+    # replace LoadImageFromFileMono3D
+    cfg_pretty_text = cfg_pretty_text.replace(
+        'LoadImageFromFileMono3D\'',
+        'LoadImageFromFileMono3D\',' + replace_strs)
+
+    # replace LoadPointsFromFile
+    cfg_pretty_text = cfg_pretty_text.replace(
+        'LoadPointsFromFile\'', 'LoadPointsFromFile\',' + replace_strs)
+
+    # replace LoadPointsFromMultiSweeps
+    cfg_pretty_text = cfg_pretty_text.replace(
+        'LoadPointsFromMultiSweeps\'',
+        'LoadPointsFromMultiSweeps\',' + replace_strs)
+
+    # replace LoadAnnotations
+    cfg_pretty_text = cfg_pretty_text.replace(
+        'LoadAnnotations\'', 'LoadAnnotations\',' + replace_strs)
+
+    # replace LoadAnnotations3D
+    cfg_pretty_text = cfg_pretty_text.replace(
+        'LoadAnnotations3D\'', 'LoadAnnotations3D\',' + replace_strs)
+
+    # replace dbsampler
+    cfg_pretty_text = cfg_pretty_text.replace('info_path',
+                                              replace_strs + ', info_path')
+
+    cfg = cfg.fromstring(cfg_pretty_text, file_format='.py')
+    return cfg

tests/test_core/test_post_processing/test_merge_augs.py

-# Copyright (c) OpenMMLab. All rights reserved.
-import mmcv
-import pytest
-import torch
-
-from mmdet3d.models import merge_aug_bboxes_3d
-from mmdet3d.structures import DepthInstance3DBoxes
-
-
-def test_merge_aug_bboxes_3d():
-    if not torch.cuda.is_available():
-        pytest.skip('test requires GPU and torch+cuda')
-    img_meta_0 = dict(
-        pcd_horizontal_flip=False,
-        pcd_vertical_flip=True,
-        pcd_scale_factor=1.0)
-    img_meta_1 = dict(
-        pcd_horizontal_flip=True,
-        pcd_vertical_flip=False,
-        pcd_scale_factor=1.0)
-    img_meta_2 = dict(
-        pcd_horizontal_flip=False,
-        pcd_vertical_flip=False,
-        pcd_scale_factor=0.5)
-    img_metas = [[img_meta_0], [img_meta_1], [img_meta_2]]
-    boxes_3d = DepthInstance3DBoxes(
-        torch.tensor(
-            [[1.0473, 4.1687, -1.2317, 2.3021, 1.8876, 1.9696, 1.6956],
-             [2.5831, 4.8117, -1.2733, 0.5852, 0.8832, 0.9733, 1.6500],
-             [-1.0864, 1.9045, -1.2000, 0.7128, 1.5631, 2.1045, 0.1022]],
-            device='cuda'))
-    labels_3d = torch.tensor([0, 7, 6], device='cuda')
-    scores_3d_1 = torch.tensor([0.3, 0.6, 0.9], device='cuda')
-    scores_3d_2 = torch.tensor([0.2, 0.5, 0.8], device='cuda')
-    scores_3d_3 = torch.tensor([0.1, 0.4, 0.7], device='cuda')
-    aug_result_1 = dict(
-        boxes_3d=boxes_3d, labels_3d=labels_3d, scores_3d=scores_3d_1)
-    aug_result_2 = dict(
-        boxes_3d=boxes_3d, labels_3d=labels_3d, scores_3d=scores_3d_2)
-    aug_result_3 = dict(
-        boxes_3d=boxes_3d, labels_3d=labels_3d, scores_3d=scores_3d_3)
-    aug_results = [aug_result_1, aug_result_2, aug_result_3]
-    test_cfg = mmcv.ConfigDict(
-        use_rotate_nms=True,
-        nms_across_levels=False,
-        nms_thr=0.01,
-        score_thr=0.1,
-        min_bbox_size=0,
-        nms_pre=100,
-        max_num=50)
-    results = merge_aug_bboxes_3d(aug_results, img_metas, test_cfg)
-    expected_boxes_3d = torch.tensor(
-        [[-1.0864, -1.9045, -1.2000, 0.7128, 1.5631, 2.1045, -0.1022],
-         [1.0864, 1.9045, -1.2000, 0.7128, 1.5631, 2.1045, 3.0394],
-         [-2.1728, 3.8090, -2.4000, 1.4256, 3.1262, 4.2090, 0.1022],
-         [2.5831, -4.8117, -1.2733, 0.5852, 0.8832, 0.9733, -1.6500],
-         [-2.5831, 4.8117, -1.2733, 0.5852, 0.8832, 0.9733, 1.4916],
-         [5.1662, 9.6234, -2.5466, 1.1704, 1.7664, 1.9466, 1.6500],
-         [1.0473, -4.1687, -1.2317, 2.3021, 1.8876, 1.9696, -1.6956],
-         [-1.0473, 4.1687, -1.2317, 2.3021, 1.8876, 1.9696, 1.4460],
-         [2.0946, 8.3374, -2.4634, 4.6042, 3.7752, 3.9392, 1.6956]])
-    expected_scores_3d = torch.tensor(
-        [0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1])
-    expected_labels_3d = torch.tensor([6, 6, 6, 7, 7, 7, 0, 0, 0])
-    assert torch.allclose(results['boxes_3d'].tensor, expected_boxes_3d)
-    assert torch.allclose(results['scores_3d'], expected_scores_3d)
-    assert torch.all(results['labels_3d'] == expected_labels_3d)

tests/test_data/test_datasets/test_s3dis_dataset.py → tests/test_datasets/test_s3dis_dataset.py

@@ -33,7 +33,10 @@ def _generate_s3dis_seg_dataset_config():
             with_label_3d=False,
             with_mask_3d=False,
             with_seg_3d=True),
-        dict(type='PointSegClassMapping'),
+        dict(
+            type='PointSegClassMapping',
+            valid_cat_ids=tuple(range(len(classes))),
+            max_cat_id=13),
         dict(
             type='IndoorPatchPointSample',
             num_points=5,
@@ -58,11 +61,12 @@ def _generate_s3dis_seg_dataset_config():
 class TestS3DISDataset(unittest.TestCase):
 
     def test_s3dis_seg(self):
-        np.random.seed(0)
         data_root, ann_file, classes, palette, scene_idxs, data_prefix, \
             pipeline, modality, = _generate_s3dis_seg_dataset_config()
 
         register_all_modules()
+        np.random.seed(0)
+
         s3dis_seg_dataset = S3DISSegDataset(
             data_root,
             ann_file,

tests/test_data/test_datasets/test_scannet_dataset.py → tests/test_datasets/test_scannet_dataset.py

@@ -55,7 +55,11 @@ def _generate_scannet_seg_dataset_config():
             with_label_3d=False,
             with_mask_3d=False,
             with_seg_3d=True),
-        dict(type='PointSegClassMapping'),
+        dict(
+            type='PointSegClassMapping',
+            valid_cat_ids=(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24,
+                           28, 33, 34, 36, 39),
+            max_cat_id=40),
         dict(
             type='IndoorPatchPointSample',
             num_points=5,
@@ -161,11 +165,11 @@ class TestScanNetDataset(unittest.TestCase):
         self.assertEqual(len(no_class_scannet_dataset.metainfo['CLASSES']), 1)
 
     def test_scannet_seg(self):
-        np.random.seed(0)
         data_root, ann_file, classes, palette, scene_idxs, data_prefix, \
             pipeline, modality, = _generate_scannet_seg_dataset_config()
 
         register_all_modules()
+        np.random.seed(0)
         scannet_seg_dataset = ScanNetSegDataset(
             data_root,
             ann_file,

tests/test_data/test_datasets/test_semantickitti_dataset.py → tests/test_datasets/test_semantickitti_dataset.py

@@ -64,11 +64,11 @@ def _generate_semantickitti_dataset_config():
 class TestSemanticKITTIDataset(unittest.TestCase):
 
     def test_semantickitti(self):
-        np.random.seed(0)
         data_root, ann_file, classes, palette, data_prefix, \
             pipeline, modality, = _generate_semantickitti_dataset_config()
 
         register_all_modules()
+        np.random.seed(0)
         semantickitti_dataset = SemanticKITTIDataset(
             data_root,
             ann_file,