[Refactor] update data flow and ut (#1776)

* update data flow and ut

* update ut

* update code

* fix mapping bug

* fix comments

configs/_base_/datasets/kitti-mono3d.py

@@ -26,8 +26,8 @@ train_pipeline = [
     dict(
         type='Pack3DDetInputs',
         keys=[
-            'img', 'gt_bboxes', 'gt_labels', 'gt_bboxes_3d', 'gt_labels_3d',
-            'centers_2d', 'depths'
+            'img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_bboxes_3d',
+            'gt_labels_3d', 'centers_2d', 'depths'
         ]),
 ]
 test_pipeline = [

configs/_base_/datasets/nus-mono3d.py

@@ -37,8 +37,8 @@ train_pipeline = [
     dict(
         type='Pack3DDetInputs',
         keys=[
-            'img', 'gt_bboxes', 'gt_labels', 'attr_labels', 'gt_bboxes_3d',
-            'gt_labels_3d', 'centers_2d', 'depths'
+            'img', 'gt_bboxes', 'gt_bboxes_labels', 'attr_labels',
+            'gt_bboxes_3d', 'gt_labels_3d', 'centers_2d', 'depths'
         ]),
 ]
 

configs/fcos3d/fcos3d_r101-caffe-dcn_fpn_head-gn_8xb2-1x_nus-mono3d.py

@@ -42,8 +42,8 @@ train_pipeline = [
     dict(
         type='Pack3DDetInputs',
         keys=[
-            'img', 'gt_bboxes', 'gt_labels', 'attr_labels', 'gt_bboxes_3d',
-            'gt_labels_3d', 'centers_2d', 'depths'
+            'img', 'gt_bboxes', 'gt_bboxes_labels', 'attr_labels',
+            'gt_bboxes_3d', 'gt_labels_3d', 'centers_2d', 'depths'
         ]),
 ]
 test_pipeline = [

configs/pgd/pgd_r101-caffe_fpn_head-gn_16xb2-1x_nus-mono3d.py

@@ -63,8 +63,8 @@ train_pipeline = [
     dict(
         type='Collect3D',
         keys=[
-            'img', 'gt_bboxes', 'gt_labels', 'attr_labels', 'gt_bboxes_3d',
-            'gt_labels_3d', 'centers2d', 'depths'
+            'img', 'gt_bboxes', 'gt_bboxes_labels', 'attr_labels',
+            'gt_bboxes_3d', 'gt_labels_3d', 'centers2d', 'depths'
         ]),
 ]
 test_pipeline = [

configs/pgd/pgd_r101-caffe_fpn_head-gn_4xb3-4x_kitti-mono3d.py

@@ -96,8 +96,8 @@ train_pipeline = [
     dict(
         type='Pack3DDetInputs',
         keys=[
-            'img', 'gt_bboxes', 'gt_labels', 'gt_bboxes_3d', 'gt_labels_3d',
-            'centers_2d', 'depths'
+            'img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_bboxes_3d',
+            'gt_labels_3d', 'centers_2d', 'depths'
         ]),
 ]
 test_pipeline = [

configs/smoke/smoke_dla34_dlaneck_gn-all_4xb8-6x_kitti-mono3d.py

@@ -32,8 +32,8 @@ train_pipeline = [
     dict(
         type='Pack3DDetInputs',
         keys=[
-            'img', 'gt_bboxes', 'gt_labels', 'gt_bboxes_3d', 'gt_labels_3d',
-            'centers_2d', 'depths'
+            'img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_bboxes_3d',
+            'gt_labels_3d', 'centers_2d', 'depths'
         ]),
 ]
 test_pipeline = [

mmdet3d/apis/inference.py

@@ -250,7 +250,7 @@ def inference_multi_modality_detector(model: nn.Module,
         results = model.test_step(data)
 
     for index in range(len(data)):
-        meta_info = data[index]['data_sample'].metainfo
+        meta_info = data[index]['data_samples'].metainfo
         results[index].set_metainfo(meta_info)
 
     if not is_batch:
@@ -320,7 +320,7 @@ def inference_mono_3d_detector(model: nn.Module,
         results = model.test_step(data)
 
     for index in range(len(data)):
-        meta_info = data[index]['data_sample'].metainfo
+        meta_info = data[index]['data_samples'].metainfo
         results[index].set_metainfo(meta_info)
 
     if not is_batch:

mmdet3d/datasets/det3d_dataset.py

@@ -187,12 +187,13 @@ class Det3DDataset(BaseDataset):
         # in `transforms`
         name_mapping = {
             'bbox_label_3d': 'gt_labels_3d',
+            'bbox_label': 'gt_bboxes_labels',
+            'bbox': 'gt_bboxes',
             'bbox_3d': 'gt_bboxes_3d',
             'depth': 'depths',
             'center_2d': 'centers_2d',
             'attr_label': 'attr_labels'
         }
-
         instances = info['instances']
         # empty gt
         if len(instances) == 0:
@@ -203,13 +204,18 @@ class Det3DDataset(BaseDataset):
             for ann_name in keys:
                 temp_anns = [item[ann_name] for item in instances]
                 # map the original dataset label to training label
-                if 'label' in ann_name:
+                if 'label' in ann_name and ann_name != 'attr_label':
                     temp_anns = [
                         self.label_mapping[item] for item in temp_anns
                     ]
                 if ann_name in name_mapping:
                     ann_name = name_mapping[ann_name]
-                temp_anns = np.array(temp_anns)
+
+                if 'label' in ann_name:
+                    temp_anns = np.array(temp_anns).astype(np.int64)
+                else:
+                    temp_anns = np.array(temp_anns).astype(np.float32)
+
                 ann_info[ann_name] = temp_anns
             ann_info['instances'] = info['instances']
         return ann_info
@@ -310,7 +316,7 @@ class Det3DDataset(BaseDataset):
             # after pipeline drop the example with empty annotations
             # return None to random another in `__getitem__`
             if example is None or len(
-                    example['data_sample'].gt_instances_3d.labels_3d) == 0:
+                    example['data_samples'].gt_instances_3d.labels_3d) == 0:
                 return None
         return example
 

mmdet3d/datasets/transforms/formating.py

@@ -97,8 +97,8 @@ class Pack3DDetInputs(BaseTransform):
                 - points
                 - img
 
-            - 'data_sample' (obj:`Det3DDataSample`): The annotation info of the
-              sample.
+            - 'data_samples' (obj:`Det3DDataSample`): The annotation info of
+                the sample.
         """
         # augtest
         if isinstance(results, list):
@@ -131,8 +131,8 @@ class Pack3DDetInputs(BaseTransform):
                 - points
                 - img
 
-            - 'data_sample' (obj:`Det3DDataSample`): The annotation info of the
-              sample.
+            - 'data_samples' (obj:`Det3DDataSample`): The annotation info
+              of the sample.
         """
         # Format 3D data
         if 'points' in results:
@@ -213,7 +213,7 @@ class Pack3DDetInputs(BaseTransform):
             data_sample.eval_ann_info = None
 
         packed_results = dict()
-        packed_results['data_sample'] = data_sample
+        packed_results['data_samples'] = data_sample
         packed_results['inputs'] = inputs
 
         return packed_results

mmdet3d/datasets/transforms/loading.py

@@ -746,17 +746,8 @@ class LoadAnnotations3D(LoadAnnotations):
         Returns:
             dict: The dict contains loaded bounding box annotations.
         """
-        gt_bboxes = []
-        for instance in results['instances']:
-            gt_bboxes.append(instance['bbox'])
-        if len(gt_bboxes) == 0:
-            results['gt_bboxes'] = np.zeros((0, 4), dtype=np.float32)
-        else:
-            results['gt_bboxes'] = np.array(
-                gt_bboxes, dtype=np.float32).reshape((-1, 4))
 
-        if 'eval_ann_info' in results:
-            results['eval_ann_info']['gt_bboxes'] = results['gt_bboxes']
+        results['gt_bboxes'] = results['ann_info']['gt_bboxes']
 
     def _load_labels(self, results: dict) -> None:
         """Private function to load label annotations.
@@ -767,17 +758,7 @@ class LoadAnnotations3D(LoadAnnotations):
         Returns:
             dict: The dict contains loaded label annotations.
         """
-        gt_bboxes_labels = []
-        for instance in results['instances']:
-            gt_bboxes_labels.append(instance['bbox_label'])
-        if len(gt_bboxes_labels) == 0:
-            results['gt_bboxes_labels'] = np.zeros((0, ), dtype=np.int64)
-        else:
-            results['gt_bboxes_labels'] = np.array(
-                gt_bboxes_labels, dtype=np.int64)
-        if 'eval_ann_info' in results:
-            results['eval_ann_info']['gt_bboxes_labels'] = results[
-                'gt_bboxes_labels']
+        results['gt_bboxes_labels'] = results['ann_info']['gt_bboxes_labels']
 
     def transform(self, results: dict) -> dict:
         """Function to load multiple types annotations.

mmdet3d/evaluation/metrics/indoor_metric.py

@@ -37,8 +37,7 @@ class IndoorMetric(BaseMetric):
             prefix=prefix, collect_device=collect_device)
         self.iou_thr = iou_thr
 
-    def process(self, data_batch: Sequence[dict],
-                predictions: Sequence[dict]) -> None:
+    def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None:
         """Process one batch of data samples and predictions.
 
         The processed results should be stored in ``self.results``,
@@ -46,23 +45,20 @@ class IndoorMetric(BaseMetric):
         have been processed.
 
         Args:
-            data_batch (Sequence[dict]): A batch of data
-                from the dataloader.
-            predictions (Sequence[dict]): A batch of outputs from
+            data_batch (dict): A batch of data from the dataloader.
+            data_samples (Sequence[dict]): A batch of outputs from
                 the model.
         """
-        batch_eval_anns = [
-            item['data_sample']['eval_ann_info'] for item in data_batch
-        ]
-        for eval_ann, pred_dict in zip(batch_eval_anns, predictions):
-            pred_3d = pred_dict['pred_instances_3d']
+        for data_sample in data_samples:
+            pred_3d = data_sample['pred_instances_3d']
+            eval_ann_info = data_sample['eval_ann_info']
             cpu_pred_3d = dict()
             for k, v in pred_3d.items():
                 if hasattr(v, 'to'):
                     cpu_pred_3d[k] = v.to('cpu')
                 else:
                     cpu_pred_3d[k] = v
-            self.results.append((eval_ann, cpu_pred_3d))
+            self.results.append((eval_ann_info, cpu_pred_3d))
 
     def compute_metrics(self, results: list) -> Dict[str, float]:
         """Compute the metrics from processed results.
@@ -121,8 +117,7 @@ class Indoor2DMetric(BaseMetric):
             prefix=prefix, collect_device=collect_device)
         self.iou_thr = iou_thr
 
-    def process(self, data_batch: Sequence[dict],
-                predictions: Sequence[dict]) -> None:
+    def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None:
         """Process one batch of data samples and predictions.
 
         The processed results should be stored in ``self.results``,
@@ -130,19 +125,16 @@ class Indoor2DMetric(BaseMetric):
         have been processed.
 
         Args:
-            data_batch (Sequence[dict]): A batch of data
-                from the dataloader.
+            data_batch (dict): A batch of data from the dataloader.
             predictions (Sequence[dict]): A batch of outputs from
                 the model.
         """
-        batch_eval_anns = [
-            item['data_sample']['eval_ann_info'] for item in data_batch
-        ]
-        for eval_ann, pred_dict in zip(batch_eval_anns, predictions):
-            pred = pred_dict['pred_instances']
+        for data_sample in data_samples:
+            pred = data_sample['pred_instances']
+            eval_ann_info = data_sample['eval_ann_info']
             ann = dict(
-                labels=eval_ann['gt_bboxes_labels'],
-                bboxes=eval_ann['gt_bboxes'])
+                labels=eval_ann_info['gt_bboxes_labels'],
+                bboxes=eval_ann_info['gt_bboxes'])
 
             pred_bboxes = pred['bboxes'].cpu().numpy()
             pred_scores = pred['scores'].cpu().numpy()

mmdet3d/evaluation/metrics/instance_seg_metric.py

@@ -29,8 +29,7 @@ class InstanceSegMetric(BaseMetric):
         super(InstanceSegMetric, self).__init__(
             prefix=prefix, collect_device=collect_device)
 
-    def process(self, data_batch: Sequence[dict],
-                predictions: Sequence[dict]) -> None:
+    def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None:
         """Process one batch of data samples and predictions.
 
         The processed results should be stored in ``self.results``,
@@ -38,23 +37,20 @@ class InstanceSegMetric(BaseMetric):
         have been processed.
 
         Args:
-            data_batch (Sequence[dict]): A batch of data
-                from the dataloader.
-            predictions (Sequence[dict]): A batch of outputs from
+            data_batch (dict): A batch of data from the dataloader.
+            data_samples (Sequence[dict]): A batch of outputs from
                 the model.
         """
-        batch_eval_anns = [
-            item['data_sample']['eval_ann_info'] for item in data_batch
-        ]
-        for eval_ann, pred_dict in zip(batch_eval_anns, predictions):
-            pred_3d = pred_dict['pred_pts_seg']
+        for data_sample in data_samples:
+            pred_3d = data_sample['pred_pts_seg']
+            eval_ann_info = data_sample['eval_ann_info']
             cpu_pred_3d = dict()
             for k, v in pred_3d.items():
                 if hasattr(v, 'to'):
                     cpu_pred_3d[k] = v.to('cpu')
                 else:
                     cpu_pred_3d[k] = v
-            self.results.append((eval_ann, cpu_pred_3d))
+            self.results.append((eval_ann_info, cpu_pred_3d))
 
     def compute_metrics(self, results: list) -> Dict[str, float]:
         """Compute the metrics from processed results.

mmdet3d/evaluation/metrics/kitti_metric.py

@@ -137,8 +137,7 @@ class KittiMetric(BaseMetric):
             data_annos[i]['kitti_annos'] = kitti_annos
         return data_annos
 
-    def process(self, data_batch: Sequence[dict],
-                predictions: Sequence[dict]) -> None:
+    def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None:
         """Process one batch of data samples and predictions.
 
         The processed results should be stored in ``self.results``,
@@ -146,20 +145,22 @@ class KittiMetric(BaseMetric):
         have been processed.
 
         Args:
-            data_batch (Sequence[dict]): A batch of data
-                from the dataloader.
-            predictions (Sequence[dict]): A batch of outputs from
+            data_batch (dict): A batch of data from the dataloader.
+            data_samples (Sequence[dict]): A batch of outputs from
                 the model.
         """
-        assert len(data_batch) == len(predictions)
-        for data, pred in zip(data_batch, predictions):
+
+        for data_sample in data_samples:
             result = dict()
-            for pred_result in pred:
-                for attr_name in pred[pred_result]:
-                    pred[pred_result][attr_name] = pred[pred_result][
-                        attr_name].to(self.collect_device)
-                result[pred_result] = pred[pred_result]
-            sample_idx = data['data_sample']['sample_idx']
+            pred_3d = data_sample['pred_instances_3d']
+            pred_2d = data_sample['pred_instances']
+            for attr_name in pred_3d:
+                pred_3d[attr_name] = pred_3d[attr_name].to('cpu')
+            result['pred_instances_3d'] = pred_3d
+            for attr_name in pred_2d:
+                pred_2d[attr_name] = pred_2d[attr_name].to('cpu')
+            result['pred_instances'] = pred_2d
+            sample_idx = data_sample['sample_idx']
             result['sample_idx'] = sample_idx
         self.results.append(result)
 

mmdet3d/evaluation/metrics/lyft_metric.py

@@ -72,30 +72,29 @@ class LyftMetric(BaseMetric):
         self.csv_savepath = csv_savepath
         self.metrics = metric if isinstance(metric, list) else [metric]
 
-    def process(self, data_batch: Sequence[dict],
-                predictions: Sequence[dict]) -> None:
-        """Process one batch of data samples and predictions.
+    def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None:
+        """Process one batch of data samples and data_samples.
 
         The processed results should be stored in ``self.results``,
         which will be used to compute the metrics when all batches
         have been processed.
 
         Args:
-            data_batch (Sequence[dict]): A batch of data
-                from the dataloader.
-            predictions (Sequence[dict]): A batch of outputs from
+            data_batch (dict): A batch of data from the dataloader.
+            data_samples (Sequence[dict]): A batch of outputs from
                 the model.
         """
-        assert len(data_batch) == len(predictions)
-        for data, pred in zip(data_batch, predictions):
+        for data_sample in data_samples:
             result = dict()
-            for pred_result in pred:
-                if pred[pred_result] is not None:
-                    for attr_name in pred[pred_result]:
-                        pred[pred_result][attr_name] = pred[pred_result][
-                            attr_name].to(self.collect_device)
-                    result[pred_result] = pred[pred_result]
-                sample_idx = data['data_sample']['sample_idx']
+            pred_3d = data_sample['pred_instances_3d']
+            pred_2d = data_sample['pred_instances']
+            for attr_name in pred_3d:
+                pred_3d[attr_name] = pred_3d[attr_name].to('cpu')
+            result['pred_instances_3d'] = pred_3d
+            for attr_name in pred_2d:
+                pred_2d[attr_name] = pred_2d[attr_name].to('cpu')
+            result['pred_instances'] = pred_2d
+            sample_idx = data_sample['sample_idx']
             result['sample_idx'] = sample_idx
         self.results.append(result)
 

mmdet3d/evaluation/metrics/nuscenes_metric.py

@@ -113,8 +113,7 @@ class NuScenesMetric(BaseMetric):
         self.eval_version = eval_version
         self.eval_detection_configs = config_factory(self.eval_version)
 
-    def process(self, data_batch: Sequence[dict],
-                predictions: Sequence[dict]) -> None:
+    def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None:
         """Process one batch of data samples and predictions.
 
         The processed results should be stored in ``self.results``,
@@ -122,21 +121,21 @@ class NuScenesMetric(BaseMetric):
         have been processed.
 
         Args:
-            data_batch (Sequence[dict]): A batch of data
-                from the dataloader.
-            predictions (Sequence[dict]): A batch of outputs from
+            data_batch (dict): A batch of data from the dataloader.
+            data_samples (Sequence[dict]): A batch of outputs from
                 the model.
         """
-        assert len(data_batch) == len(predictions)
-        for data, pred in zip(data_batch, predictions):
+        for data_sample in data_samples:
             result = dict()
-            for pred_result in pred:
-                if pred[pred_result] is not None:
-                    for attr_name in pred[pred_result]:
-                        pred[pred_result][attr_name] = pred[pred_result][
-                            attr_name].to(self.collect_device)
-                    result[pred_result] = pred[pred_result]
-                sample_idx = data['data_sample']['sample_idx']
+            pred_3d = data_sample['pred_instances_3d']
+            pred_2d = data_sample['pred_instances']
+            for attr_name in pred_3d:
+                pred_3d[attr_name] = pred_3d[attr_name].to('cpu')
+            result['pred_instances_3d'] = pred_3d
+            for attr_name in pred_2d:
+                pred_2d[attr_name] = pred_2d[attr_name].to('cpu')
+            result['pred_instances'] = pred_2d
+            sample_idx = data_sample['sample_idx']
             result['sample_idx'] = sample_idx
         self.results.append(result)
 

mmdet3d/evaluation/metrics/seg_metric.py

@@ -29,8 +29,7 @@ class SegMetric(BaseMetric):
         super(SegMetric, self).__init__(
             prefix=prefix, collect_device=collect_device)
 
-    def process(self, data_batch: Sequence[dict],
-                predictions: Sequence[dict]) -> None:
+    def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None:
         """Process one batch of data samples and predictions.
 
         The processed results should be stored in ``self.results``,
@@ -38,23 +37,20 @@ class SegMetric(BaseMetric):
         have been processed.
 
         Args:
-            data_batch (Sequence[dict]): A batch of data
-                from the dataloader.
-            predictions (Sequence[dict]): A batch of outputs from
+            data_batch (dict): A batch of data from the dataloader.
+            data_samples (Sequence[dict]): A batch of outputs from
                 the model.
         """
-        batch_eval_anns = [
-            item['data_sample']['eval_ann_info'] for item in data_batch
-        ]
-        for eval_ann, pred_dict in zip(batch_eval_anns, predictions):
-            pred_3d = pred_dict['pred_pts_seg']
+        for data_sample in data_samples:
+            pred_3d = data_sample['pred_pts_seg']
+            eval_ann_info = data_sample['eval_ann_info']
             cpu_pred_3d = dict()
             for k, v in pred_3d.items():
                 if hasattr(v, 'to'):
                     cpu_pred_3d[k] = v.to('cpu').numpy()
                 else:
                     cpu_pred_3d[k] = v
-            self.results.append((eval_ann, cpu_pred_3d))
+            self.results.append((eval_ann_info, cpu_pred_3d))
 
     def compute_metrics(self, results: list) -> Dict[str, float]:
         """Compute the metrics from processed results.

mmdet3d/models/data_preprocessors/data_preprocessor.py

 # Copyright (c) OpenMMLab. All rights reserved.
+import math
 from numbers import Number
 from typing import Dict, List, Optional, Sequence, Tuple, Union
 
@@ -6,12 +7,13 @@ import numpy as np
 import torch
 from mmcv.ops import Voxelization
 from mmengine.model import stack_batch
-from mmengine.structures import BaseDataElement
+from mmengine.utils import is_list_of
 from torch.nn import functional as F
 
 from mmdet3d.registry import MODELS
 from mmdet3d.utils import OptConfigType
 from mmdet.models import DetDataPreprocessor
+from mmdet.models.utils.misc import samplelist_boxlist2tensor
 
 
 @MODELS.register_module()
@@ -73,6 +75,7 @@ class Det3DDataPreprocessor(DetDataPreprocessor):
                  seg_pad_value: int = 255,
                  bgr_to_rgb: bool = False,
                  rgb_to_bgr: bool = False,
+                 boxlist2tensor: bool = True,
                  batch_augments: Optional[List[dict]] = None):
         super().__init__(
             mean=mean,
@@ -85,14 +88,16 @@ class Det3DDataPreprocessor(DetDataPreprocessor):
             seg_pad_value=seg_pad_value,
             bgr_to_rgb=bgr_to_rgb,
             rgb_to_bgr=rgb_to_bgr,
+            boxlist2tensor=boxlist2tensor,
             batch_augments=batch_augments)
         self.voxel = voxel
         self.voxel_type = voxel_type
         if voxel:
             self.voxel_layer = Voxelization(**voxel_layer)
 
-    def forward(self,
-                data: List[Union[dict, List[dict]]],
+    def forward(
+        self,
+        data: Union[dict, List[dict]],
         training: bool = False
     ) -> Tuple[Union[dict, List[dict]], Optional[list]]:
         """Perform normalization、padding and bgr2rgb conversion based on
@@ -104,134 +109,191 @@ class Det3DDataPreprocessor(DetDataPreprocessor):
                 a list[list[dict]], the inter list indicate test time
                 augmentation.
             training (bool): Whether to enable training time augmentation.
+                Defaults to False.
 
         Returns:
-            Tuple[Dict, Optional[list]] |
-            Tuple[List[Dict], Optional[list[list]]]:
-            Data in the same format as the model input.
+            Dict | List[Dict]: Data in the same format as the model input.
         """
-        if isinstance(data[0], list):
-            num_augs = len(data[0])
+        if isinstance(data, list):
+            num_augs = len(data)
             aug_batch_data = []
-            aug_batch_data_sample = []
             for aug_id in range(num_augs):
-                single_aug_batch_data, \
-                    single_aug_batch_data_sample = self.simple_process(
-                        [item[aug_id] for item in data], training)
+                single_aug_batch_data = self.simple_process(
+                    data[aug_id], training)
                 aug_batch_data.append(single_aug_batch_data)
-                aug_batch_data_sample.append(single_aug_batch_data_sample)
-
-            return aug_batch_data, aug_batch_data_sample
+            return aug_batch_data
 
         else:
             return self.simple_process(data, training)
 
-    def simple_process(self, data: Sequence[dict], training: bool = False):
-        inputs_dict, batch_data_samples = self.collate_data(data)
+    def simple_process(self, data: dict, training: bool = False) -> dict:
+        """Perform normalization、padding and bgr2rgb conversion for img data
+        based on ``BaseDataPreprocessor``, and voxelize point cloud if `voxel`
+        is set to be True.
 
-        if 'points' in inputs_dict[0].keys():
-            points = [input['points'] for input in inputs_dict]
-        else:
-            points = None
+        Args:
+            data (dict): Data sampled from dataloader.
+            training (bool): Whether to enable training time augmentation.
+                Defaults to False.
 
-        if 'img' in inputs_dict[0].keys():
+        Returns:
+            dict: Data in the same format as the model input.
+        """
+        if 'img' in data['inputs']:
+            batch_pad_shape = self._get_pad_shape(data)
 
-            imgs = [input['img'] for input in inputs_dict]
+        data = self.collate_data(data)
+        inputs, data_samples = data['inputs'], data['data_samples']
 
-            # channel transform
-            if self.channel_conversion:
-                imgs = [_img[[2, 1, 0], ...] for _img in imgs]
-            # Normalization.
-            if self._enable_normalize:
-                imgs = [(_img.float() - self.mean) / self.std for _img in imgs]
-            # Pad and stack Tensor.
-            batch_imgs = stack_batch(imgs, self.pad_size_divisor,
-                                     self.pad_value)
+        batch_inputs = dict()
 
-            batch_pad_shape = self._get_pad_shape(data)
+        if 'points' in inputs:
+            batch_inputs['points'] = inputs['points']
 
-            if batch_data_samples is not None:
+            if self.voxel:
+                voxel_dict = self.voxelize(inputs['points'])
+                batch_inputs['voxels'] = voxel_dict
+
+        if 'imgs' in inputs:
+            imgs = inputs['imgs']
+
+            if data_samples is not None:
                 # NOTE the batched image size information may be useful, e.g.
-                batch_input_shape = tuple(batch_imgs[0].size()[-2:])
-                for data_samples, pad_shape in zip(batch_data_samples,
+                # in DETR, this is needed for the construction of masks, which
+                # is then used for the transformer_head.
+                batch_input_shape = tuple(imgs[0].size()[-2:])
+                for data_sample, pad_shape in zip(data_samples,
                                                   batch_pad_shape):
-                    data_samples.set_metainfo({
+                    data_sample.set_metainfo({
                         'batch_input_shape': batch_input_shape,
                         'pad_shape': pad_shape
                     })
 
+                if self.boxlist2tensor:
+                    samplelist_boxlist2tensor(data_samples)
+
                 if self.pad_mask:
-                    self.pad_gt_masks(batch_data_samples)
+                    self.pad_gt_masks(data_samples)
 
                 if self.pad_seg:
-                    self.pad_gt_sem_seg(batch_data_samples)
+                    self.pad_gt_sem_seg(data_samples)
 
             if training and self.batch_augments is not None:
                 for batch_aug in self.batch_augments:
-                    batch_imgs, batch_data_samples = batch_aug(
-                        batch_imgs, batch_data_samples)
-        else:
-            imgs = None
+                    imgs, data_samples = batch_aug(imgs, data_samples)
+            batch_inputs['imgs'] = imgs
 
-        batch_inputs_dict = {
-            'points': points,
-            'imgs': batch_imgs if imgs is not None else None
-        }
+        return {'inputs': batch_inputs, 'data_samples': data_samples}
 
-        if self.voxel:
-            voxel_dict = self.voxelize(points)
-            batch_inputs_dict['voxels'] = voxel_dict
-
-        return batch_inputs_dict, batch_data_samples
-
-    def collate_data(
-            self, data: Sequence[dict]) -> Tuple[List[dict], Optional[list]]:
-        """Collating and copying data to the target device.
+    def collate_data(self, data: dict) -> dict:
+        """Copying data to the target device and Performs normalization、
+        padding and bgr2rgb conversion and stack based on
+        ``BaseDataPreprocessor``.
 
         Collates the data sampled from dataloader into a list of dict and
         list of labels, and then copies tensor to the target device.
 
         Args:
-            data (Sequence[dict]): Data sampled from dataloader.
+            data (dict): Data sampled from dataloader.
 
         Returns:
-            Tuple[List[Dict], Optional[list]]: Unstacked list of input
-            data dict and list of labels at target device.
+            dict: Data in the same format as the model input.
         """
-        # rewrite `collate_data` since the inputs is a dict instead of
-        # image tensor.
-        inputs_dict = [{
-            k: v.to(self._device)
-            for k, v in _data['inputs'].items() if v is not None
-        } for _data in data]
-
-        batch_data_samples: List[BaseDataElement] = []
-        # Model can get predictions without any data samples.
-        for _data in data:
-            if 'data_sample' in _data:
-                batch_data_samples.append(_data['data_sample'])
-        # Move data from CPU to corresponding device.
-        batch_data_samples = [
-            data_sample.to(self._device) for data_sample in batch_data_samples
-        ]
-
-        if not batch_data_samples:
-            batch_data_samples = None  # type: ignore
-
-        return inputs_dict, batch_data_samples
-
-    def _get_pad_shape(self, data: Sequence[dict]) -> List[tuple]:
+        data = self.cast_data(data)  # type: ignore
+
+        if 'img' in data['inputs']:
+            _batch_imgs = data['inputs']['img']
+
+            # Process data with `pseudo_collate`.
+            if is_list_of(_batch_imgs, torch.Tensor):
+                batch_imgs = []
+                for _batch_img in _batch_imgs:
+                    # channel transform
+                    if self._channel_conversion:
+                        _batch_img = _batch_img[[2, 1, 0], ...]
+                    # Convert to float after channel conversion to ensure
+                    # efficiency
+                    _batch_img = _batch_img.float()
+                    # Normalization.
+                    if self._enable_normalize:
+                        if self.mean.shape[0] == 3:
+                            assert _batch_img.dim(
+                            ) == 3 and _batch_img.shape[0] == 3, (
+                                'If the mean has 3 values, the input tensor '
+                                'should in shape of (3, H, W), but got the '
+                                f'tensor with shape {_batch_img.shape}')
+                        _batch_img = (_batch_img - self.mean) / self.std
+                    batch_imgs.append(_batch_img)
+                # Pad and stack Tensor.
+                batch_imgs = stack_batch(batch_imgs, self.pad_size_divisor,
+                                         self.pad_value)
+            # Process data with `default_collate`.
+            elif isinstance(_batch_imgs, torch.Tensor):
+                assert _batch_imgs.dim() == 4, (
+                    'The input of `ImgDataPreprocessor` should be a NCHW '
+                    'tensor or a list of tensor, but got a tensor with '
+                    f'shape: {_batch_imgs.shape}')
+                if self._channel_conversion:
+                    _batch_imgs = _batch_imgs[:, [2, 1, 0], ...]
+                # Convert to float after channel conversion to ensure
+                # efficiency
+                _batch_imgs = _batch_imgs.float()
+                if self._enable_normalize:
+                    _batch_imgs = (_batch_imgs - self.mean) / self.std
+                h, w = _batch_imgs.shape[2:]
+                target_h = math.ceil(
+                    h / self.pad_size_divisor) * self.pad_size_divisor
+                target_w = math.ceil(
+                    w / self.pad_size_divisor) * self.pad_size_divisor
+                pad_h = target_h - h
+                pad_w = target_w - w
+                batch_imgs = F.pad(_batch_imgs, (0, pad_w, 0, pad_h),
+                                   'constant', self.pad_value)
+            else:
+                raise TypeError(
+                    'Output of `cast_data` should be a list of dict '
+                    'or a tuple with inputs and data_samples, but got'
+                    f'{type(data)}： {data}')
+
+            data['inputs']['imgs'] = batch_imgs
+
+        data.setdefault('data_samples', None)
+
+        return data
+
+    def _get_pad_shape(self, data: dict) -> List[tuple]:
         """Get the pad_shape of each image based on data and
         pad_size_divisor."""
         # rewrite `_get_pad_shape` for obaining image inputs.
-        ori_inputs = [_data['inputs']['img'] for _data in data]
+        _batch_inputs = data['inputs']['img']
+        # Process data with `pseudo_collate`.
+        if is_list_of(_batch_inputs, torch.Tensor):
             batch_pad_shape = []
-        for ori_input in ori_inputs:
-            pad_h = int(np.ceil(ori_input.shape[1] /
+            for ori_input in _batch_inputs:
+                pad_h = int(
+                    np.ceil(ori_input.shape[1] /
                             self.pad_size_divisor)) * self.pad_size_divisor
-            pad_w = int(np.ceil(ori_input.shape[2] /
+                pad_w = int(
+                    np.ceil(ori_input.shape[2] /
                             self.pad_size_divisor)) * self.pad_size_divisor
                 batch_pad_shape.append((pad_h, pad_w))
+        # Process data with `default_collate`.
+        elif isinstance(_batch_inputs, torch.Tensor):
+            assert _batch_inputs.dim() == 4, (
+                'The input of `ImgDataPreprocessor` should be a NCHW tensor '
+                'or a list of tensor, but got a tensor with shape: '
+                f'{_batch_inputs.shape}')
+            pad_h = int(
+                np.ceil(_batch_inputs.shape[1] /
+                        self.pad_size_divisor)) * self.pad_size_divisor
+            pad_w = int(
+                np.ceil(_batch_inputs.shape[2] /
+                        self.pad_size_divisor)) * self.pad_size_divisor
+            batch_pad_shape = [(pad_h, pad_w)] * _batch_inputs.shape[0]
+        else:
+            raise TypeError('Output of `cast_data` should be a list of dict '
+                            'or a tuple with inputs and data_samples, but got'
+                            f'{type(data)}： {data}')
         return batch_pad_shape
 
     @torch.no_grad()

mmdet3d/models/dense_heads/fcos_mono3d_head.py

@@ -568,7 +568,8 @@ class FCOSMono3DHead(AnchorFreeMono3DHead):
                 cfg=cfg,
                 rescale=rescale)
             result_list.append(results)
-        return result_list
+        result_list_2d = None
+        return result_list, result_list_2d
 
     def _predict_by_feat_single(self,
                                 cls_score_list: List[Tensor],

mmdet3d/models/dense_heads/pgd_head.py

@@ -847,6 +847,8 @@ class PGDHead(FCOSMono3DHead):
         mlvl_points = self.get_points(featmap_sizes, bbox_preds[0].dtype,
                                       bbox_preds[0].device)
         result_list = []
+        result_list_2d = []
+
         for img_id in range(len(batch_img_metas)):
             cls_score_list = [
                 cls_scores[i][img_id].detach() for i in range(num_levels)
@@ -901,7 +903,7 @@ class PGDHead(FCOSMono3DHead):
                 centernesses[i][img_id].detach() for i in range(num_levels)
             ]
             img_meta = batch_img_metas[img_id]
-            results = self._predict_by_feat_single(
+            results, results_2d = self._predict_by_feat_single(
                 cls_score_list=cls_score_list,
                 bbox_pred_list=bbox_pred_list,
                 dir_cls_pred_list=dir_cls_pred_list,
@@ -914,7 +916,8 @@ class PGDHead(FCOSMono3DHead):
                 cfg=cfg,
                 rescale=rescale)
             result_list.append(results)
-        return result_list
+            result_list_2d.append(results_2d)
+        return result_list, result_list_2d
 
     def _predict_by_feat_single(self,
                                 cls_score_list: List[Tensor],
@@ -1117,15 +1120,15 @@ class PGDHead(FCOSMono3DHead):
         if attrs is not None:
             results.attr_labels = attrs
 
-        if self.pred_bbox2d:
         results_2d = InstanceData()
+
+        if self.pred_bbox2d:
             bboxes2d = nms_results[-1]
             results_2d.bboxes = bboxes2d
             results_2d.scores = scores
             results_2d.labels = labels
+
         return results, results_2d
-        else:
-            return results
 
     def get_targets(
         self,

mmdet3d/models/detectors/base.py

 # Copyright (c) OpenMMLab. All rights reserved.
-from typing import List, Optional, Union
+from typing import List, Union
 
 from mmengine.structures import InstanceData
 
 from mmdet3d.registry import MODELS
-from mmdet3d.structures import Det3DDataSample
 from mmdet3d.structures.det3d_data_sample import (ForwardResults,
                                                   OptSampleList, SampleList)
-from mmdet3d.utils.typing import InstanceList, OptConfigType, OptMultiConfig
+from mmdet3d.utils.typing import OptConfigType, OptInstanceList, OptMultiConfig
 from mmdet.models import BaseDetector
 
 
@@ -92,8 +91,9 @@ class Base3DDetector(BaseDetector):
 
     def convert_to_datasample(
         self,
-        results_list_3d: Optional[InstanceList] = None,
-        results_list_2d: Optional[InstanceList] = None,
+        data_samples: SampleList,
+        data_instances_3d: OptInstanceList = None,
+        data_instances_2d: OptInstanceList = None,
     ) -> SampleList:
         """Convert results list to `Det3DDataSample`.
 
@@ -101,8 +101,11 @@ class Base3DDetector(BaseDetector):
         3D detectors.
 
         Args:
-            results_list (list[:obj:`InstanceData`]): Detection results of
-                each sample.
+            data_samples (list[:obj:`Det3DDataSample`]): The input data.
+            data_instances_3d (list[:obj:`InstanceData`], optional): 3D
+                Detection results of each sample.
+            data_instances_2d (list[:obj:`InstanceData`], optional): 2D
+                Detection results of each sample.
 
         Returns:
             list[:obj:`Det3DDataSample`]: Detection results of the
@@ -116,6 +119,7 @@ class Base3DDetector(BaseDetector):
               (num_instances, ).
             - bboxes_3d (Tensor): Contains a tensor with shape
               (num_instances, C) where C >=7.
+
             When there are image prediction in some models, it should
             contains  `pred_instances`, And the ``pred_instances`` normally
             contains following keys.
@@ -128,22 +132,20 @@ class Base3DDetector(BaseDetector):
               (num_instances, 4).
         """
 
-        data_sample_list = []
-        assert (results_list_2d is not None) or \
-               (results_list_3d is not None),\
-               'please pass at least one type of results_list'
+        assert (data_instances_2d is not None) or \
+               (data_instances_3d is not None),\
+               'please pass at least one type of data_samples'
 
-        if results_list_2d is None:
-            results_list_2d = [
-                InstanceData() for _ in range(len(results_list_3d))
+        if data_instances_2d is None:
+            data_instances_2d = [
+                InstanceData() for _ in range(len(data_instances_3d))
             ]
-        if results_list_3d is None:
-            results_list_3d = [
-                InstanceData() for _ in range(len(results_list_2d))
+        if data_instances_3d is None:
+            data_instances_3d = [
+                InstanceData() for _ in range(len(data_instances_2d))
             ]
-        for i in range(len(results_list_3d)):
-            result = Det3DDataSample()
-            result.pred_instances_3d = results_list_3d[i]
-            result.pred_instances = results_list_2d[i]
-            data_sample_list.append(result)
-        return data_sample_list
+
+        for i, data_sample in enumerate(data_samples):
+            data_sample.pred_instances_3d = data_instances_3d[i]
+            data_sample.pred_instances = data_instances_2d[i]
+        return data_samples