Release v1.1.0rc1

Release v1.1.0rc1

demo/mono_det_demo.py

@@ -19,7 +19,7 @@ def parse_args():
     parser.add_argument(
         '--cam-type',
         type=str,
-        default='CAM_FRONT',
+        default='CAM_BACK',
         help='choose camera type to inference')
     parser.add_argument(
         '--score-thr', type=float, default=0.30, help='bbox score threshold')
@@ -66,7 +66,7 @@ def main(args):
         wait_time=0,
         out_file=args.out_dir,
         pred_score_thr=args.score_thr,
-        vis_task='mono-det')
+        vis_task='mono_det')
 
 
 if __name__ == '__main__':

demo/multi_modality_demo.py

@@ -67,7 +67,7 @@ def main(args):
         wait_time=0,
         out_file=args.out_dir,
         pred_score_thr=args.score_thr,
-        vis_task='multi_modality-det')
+        vis_task='multi-modality_det')
 
 
 if __name__ == '__main__':

demo/pcd_demo.py

@@ -56,7 +56,7 @@ def main(args):
         wait_time=0,
         out_file=args.out_dir,
         pred_score_thr=args.score_thr,
-        vis_task='det')
+        vis_task='lidar_det')
 
 
 if __name__ == '__main__':

demo/pcd_seg_demo.py

@@ -51,7 +51,7 @@ def main(args):
         show=True,
         wait_time=0,
         out_file=args.out_dir,
-        vis_task='seg')
+        vis_task='lidar_seg')
 
 
 if __name__ == '__main__':

docs/en/advanced_guides/customize_models.md

@@ -365,7 +365,7 @@ class PartAggregationROIHead(Base3DRoIHead):
 
         Args:
             feats_dict (dict): Contains features from the first stage.
-            rpn_results_list (List[:obj:`InstancesData`]): Detection results
+            rpn_results_list (List[:obj:`InstanceData`]): Detection results
                 of rpn head.
             batch_data_samples (List[:obj:`Det3DDataSample`]): The Data
                 samples. It usually includes information such as
@@ -412,7 +412,7 @@ class PartAggregationROIHead(Base3DRoIHead):
             voxel_dict (dict): Contains information of voxels.
             batch_input_metas (list[dict], Optional): Batch image meta info.
                 Defaults to None.
-            rpn_results_list (List[:obj:`InstancesData`]): Detection results
+            rpn_results_list (List[:obj:`InstanceData`]): Detection results
                 of rpn head.
             test_cfg (Config): Test config.
 
@@ -438,7 +438,7 @@ class PartAggregationROIHead(Base3DRoIHead):
 
         Args:
             feats_dict (dict): Contains features from the first stage.
-            rpn_results_list (List[:obj:`InstancesData`]): Detection results
+            rpn_results_list (List[:obj:`InstanceData`]): Detection results
                 of rpn head.
             batch_data_samples (List[:obj:`Det3DDataSample`]): The Data
                 samples. It usually includes information such as

docs/en/advanced_guides/customize_runtime.md

@@ -234,7 +234,7 @@ Actually, both [`IterBasedTrainLoop`](https://github.com/open-mmlab/mmengine/blo
 
 ```python
 # Before 365001th iteration, we do evaluation every 5000 iterations.
-# After 365000th iteration, we do evaluation every 368750 iteraions,
+# After 365000th iteration, we do evaluation every 368750 iterations,
 # which means that we do evaluation at the end of training.
 
 interval = 5000

docs/en/advanced_guides/datasets/lyft_det.md

@@ -78,38 +78,48 @@ mmdetection3d
 │   │   ├── lyft_infos_train.pkl
 │   │   ├── lyft_infos_val.pkl
 │   │   ├── lyft_infos_test.pkl
-│   │   ├── lyft_infos_train_mono3d.coco.json
-│   │   ├── lyft_infos_val_mono3d.coco.json
-│   │   ├── lyft_infos_test_mono3d.coco.json
 ```
 
-Here, .pkl files are generally used for methods involving point clouds, and coco-style .json files are more suitable for image-based methods, such as image-based 2D and 3D detection.
-Different from nuScenes, we only support using the json files for 2D detection experiments. Image-based 3D detection may be further supported in the future.
+- `lyft_infos_train.pkl`: training dataset, a dict contains two keys: `metainfo` and `data_list`.
+  `metainfo` contains the basic information for the dataset itself, such as `CLASSES` and `version`, while `data_list` is a list of dict, each dict ( hereinafter referred to as`info`) contains all the detailed information of single sample as follows:
+  - info\['sample_idx'\]: The index of this sample in the whole dataset.
+  - info\['token'\]: Sample data token.
+  - info\['timestamp'\]: Timestamp of the sample data.
+  - info\['lidar_points'\]: A dict contains all the information related to the lidar points.
+    - info\['lidar_points'\]\['lidar_path'\]: The filename of the lidar point cloud data.
+    - info\['lidar_points'\]\['lidar2ego'\]: The transformation matrix from this lidar sensor to ego vehicle. (4x4 list)
+    - info\['lidar_points'\]\['ego2global'\]: The transformation matrix from the ego vehicle to global coordinates. (4x4 list)
+  - info\['lidar_sweeps'\]: A list contains sweeps information (The intermediate lidar frames without annotations)
+    - info\['lidar_sweeps'\]\[i\]\['lidar_points'\]\['data_path'\]: The lidar data path of i-th sweep.
+    - info\['lidar_sweeps'\]\[i\]\['lidar_points'\]\['lidar2ego'\]: The transformation matrix from this lidar sensor to ego vehicle in i-th sweep timestamp
+    - info\['lidar_sweeps'\]\[i\]\['lidar_points'\]\['ego2global'\]: The transformation matrix from the ego vehicle in i-th sweep timestamp to global coordinates. (4x4 list)
+    - info\['lidar_sweeps'\]\[i\]\['lidar2sensor'\]: The transformation matrix from the the lidar (for collecting the i-th sweep data) to the lidar collecting the key/sample data. (4x4 list)
+    - info\['lidar_sweeps'\]\[i\]\['timestamp'\]: Timestamp of the sweep data.
+    - info\['lidar_sweeps'\]\[i\]\['sample_data_token'\]: The sweep sample data token.
+  - info\['images'\]: A dict contains six keys corresponding to each camera: `'CAM_FRONT'`, `'CAM_FRONT_RIGHT'`, `'CAM_FRONT_LEFT'`, `'CAM_BACK'`, `'CAM_BACK_LEFT'`, `'CAM_BACK_RIGHT'`. Each dict contains all data information related to  corresponding camera.
+    - info\['images'\]\['CAM_XXX'\]\['img_path'\]: Filename of image.
+    - info\['images'\]\['CAM_XXX'\]\['cam2img'\]: The transformation matrix recording the intrinsic parameters when projecting 3D points to each image plane. (3x3 list)
+    - info\['images'\]\['CAM_XXX'\]\['sample_data_token'\]: Sample data token of image.
+    - info\['images'\]\['CAM_XXX'\]\['timestamp'\]: Timestamp of the image.
+    - info\['images'\]\['CAM_XXX'\]\['cam2ego'\]: The transformation matrix from this camera sensor to ego vehicle. (4x4 list)
+    - info\['images'\]\['CAM_XXX'\]\['lidar2cam'\]: The transformation matrix from lidar sensor to this camera. (4x4 list)
+  - info\['instances'\]: It is a list of dict. Each dict contains all annotation information of single instance.
+    - info\['instances'\]\['bbox_3d'\]: List of 7 numbers representing the 3D bounding box in lidar coordinate system of the instance, in (x, y, z, l, w, h, yaw) order.
+    - info\['instances'\]\['bbox_label_3d'\]: A int starting from 0 indicates the label of instance, while the -1 indicates ignore class.
+    - info\['instances'\]\['bbox_3d_isvalid'\]: Whether each bounding box is valid. In general, we only take the 3D boxes that include at least one lidar or radar point as valid boxes.
 
 Next, we will elaborate on the difference compared to nuScenes in terms of the details recorded in these info files.
 
 - without `lyft_database/xxxxx.bin`: This folder and `.bin` files are not extracted on the Lyft dataset due to the negligible effect of ground-truth sampling in the experiments.
-- `lyft_infos_train.pkl`: training dataset infos, each frame info has two keys: `metadata` and `infos`.
-  `metadata` contains the basic information for the dataset itself, such as `{'version': 'v1.01-train'}`, while `infos` contains the detailed information the same as nuScenes except for the following details:
-  - info\['sweeps'\]: Sweeps information.
-    - info\['sweeps'\]\[i\]\['type'\]: The sweep data type, e.g., `'lidar'`.
-      Lyft has different LiDAR settings for some samples, but we always take only the points collected by the top LiDAR for the consistency of data distribution.
-  - info\['gt_names'\]: There are 9 categories on the Lyft dataset, and the imbalance of annotations for different categories is even more significant than nuScenes.
-  - without info\['gt_velocity'\]: There is no velocity measurement on Lyft.
-  - info\['num_lidar_pts'\]: Set to -1 by default.
-  - info\['num_radar_pts'\]: Set to 0 by default.
-  - without info\['valid_flag'\]: This flag does recorded due to invalid `num_lidar_pts` and `num_radar_pts`.
-- `nuscenes_infos_train_mono3d.coco.json`: training dataset coco-style info. This file only contains 2D information, without the information required by 3D detection, such as camera intrinsics.
-  - info\['images'\]: A list containing all the image info.
-    - only containing `'file_name'`, `'id'`, `'width'`, `'height'`.
-  - info\['annotations'\]: A list containing all the annotation info.
-    - only containing `'file_name'`, `'image_id'`, `'area'`, `'category_name'`, `'category_id'`, `'bbox'`, `'is_crowd'`, `'segmentation'`, `'id'`, where `'is_crowd'`, `'segmentation'` are set to `0` and `[]` by default.
-      There is no attribute annotation on Lyft.
-
-Here we only explain the data recorded in the training info files. The same applies to the testing set.
-
-The core function to get `lyft_infos_xxx.pkl` is [\_fill_trainval_infos](https://github.com/open-mmlab/mmdetection3d/blob/master/tools/dataset_converters/lyft_converter.py#L93).
-Please refer to [lyft_converter.py](https://github.com/open-mmlab/mmdetection3d/blob/master/tools/dataset_converters/lyft_converter.py) for more details.
+
+- `lyft_infos_train.pkl`:
+
+  - Without info\['instances'\]\['velocity'\], There is no velocity measurement on Lyft.
+  - Without info\['instances'\]\['num_lidar_pts'\] and info\['instances'\]\['num_radar_pts'\]
+
+Here we only explain the data recorded in the training info files. The same applies to the validation set and test set(without instances).
+
+Please refer to [lyft_converter.py](https://github.com/open-mmlab/mmdetection3d/blob/dev-1.x/tools/dataset_converters/lyft_converter.py) for more details about the structure of `lyft_infos_xxx.pkl`.
 
 ## Training pipeline
 
@@ -124,11 +134,11 @@ train_pipeline = [
         coord_type='LIDAR',
         load_dim=5,
         use_dim=5,
-        file_client_args=file_client_args),
+        ),
     dict(
         type='LoadPointsFromMultiSweeps',
         sweeps_num=10,
-        file_client_args=file_client_args),
+        ),
     dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True),
     dict(
         type='GlobalRotScaleTrans',
@@ -139,8 +149,9 @@ train_pipeline = [
     dict(type='PointsRangeFilter', point_cloud_range=point_cloud_range),
     dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range),
     dict(type='PointShuffle'),
-    dict(type='DefaultFormatBundle3D', class_names=class_names),
-    dict(type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d'])
+    dict(
+        type='Pack3DDetInputs',
+        keys=['points', 'gt_bboxes_3d', 'gt_labels_3d'])
 ]
 ```
 
@@ -153,7 +164,7 @@ where the first 3 dimensions refer to point coordinates, and the last refers to
 An example to evaluate PointPillars with 8 GPUs with Lyft metrics is as follows.
 
 ```shell
-bash ./tools/dist_test.sh configs/pointpillars/pointpillars_hv_fpn_sbn-all_8xb2-2x_lyft-3d.py checkpoints/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d_20210517_202818-fc6904c3.pth 8 --eval bbox
+bash ./tools/dist_test.sh configs/pointpillars/pointpillars_hv_fpn_sbn-all_8xb2-2x_lyft-3d.py checkpoints/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d_20210517_202818-fc6904c3.pth 8
 ```
 
 ## Metrics
@@ -189,7 +200,7 @@ We employ this official method for evaluation on Lyft. An example of printed eva
 An example to test PointPillars on Lyft with 8 GPUs and generate a submission to the leaderboard is as follows.
 
 ```shell
-./tools/dist_test.sh configs/pointpillars/pointpillars_hv_fpn_sbn-all_8xb2-2x_lyft-3d.py work_dirs/pp-lyft/latest.pth 8 --out work_dirs/pp-lyft/results_challenge.pkl --format-only --eval-options 'jsonfile_prefix=work_dirs/pp-lyft/results_challenge' 'csv_savepath=results/pp-lyft/results_challenge.csv'
+./tools/dist_test.sh configs/pointpillars/pointpillars_hv_fpn_sbn-all_8xb2-2x_lyft-3d.py work_dirs/pp-lyft/latest.pth 8 --cfg-options test_evaluator.jsonfile_prefix=work_dirs/pp-lyft/results_challenge  test_evaluator.csv_savepath=results/pp-lyft/results_challenge.csv
 ```
 
 After generating the `work_dirs/pp-lyft/results_challenge.csv`, you can submit it to the Kaggle evaluation server. Please refer to the [official website](https://www.kaggle.com/c/3d-object-detection-for-autonomous-vehicles) for more information.

docs/en/advanced_guides/datasets/scannet_det.md

@@ -226,11 +226,12 @@ scannet
 - `semantic_mask/xxxxx.bin`: The semantic label for each point, value range: \[1, 40\], i.e. `nyu40id` standard. Note: the `nyu40id` ID will be mapped to train ID in train pipeline `PointSegClassMapping`.
 - `posed_images/scenexxxx_xx`: The set of `.jpg` images with `.txt` 4x4 poses and the single `.txt` file with camera intrinsic matrix.
 - `scannet_infos_train.pkl`: The train data infos, the detailed info of each scan is as follows:
+  - info\['lidar_points'\]: A dict contains all information relate to the the lidar points.
     - info\['lidar_points'\]\['lidar_path'\]: The filename of `xxx.bin` of lidar points.
     - info\['lidar_points'\]\['num_pts_feats'\]: The feature dimension of point.
-  - info\['lidar_points'\]\['pts_semantic_mask_path'\]: The filename of `xxx.bin` contains semantic mask annotation.
-  - info\['lidar_points'\]\['pts_instance_mask_path'\]: The filename of `xxx.bin` contains semantic mask annotation.
     - info\['lidar_points'\]\['axis_align_matrix'\]: The transformation matrix to align the axis.
+  - info\['pts_semantic_mask_path'\]: The filename of `xxx.bin` contains semantic mask annotation.
+  - info\['pts_instance_mask_path'\]: The filename of `xxx.bin` contains semantic mask annotation.
   - info\['instances'\]: A list of dict contains all annotations, each dict contains all annotation information of single instance.
     - info\['instances'\]\[i\]\['bbox_3d'\]: List of 6 numbers representing the axis-aligned 3D bounding box of the instance in depth coordinate system, in (x, y, z, l, w, h) order.
     - info\['instances'\]\[i\]\['bbox_label_3d'\]: The label of each 3d bounding boxes.

docs/en/advanced_guides/datasets/sunrgbd_det.md

@@ -133,94 +133,7 @@ or (if in a slurm environment)
 bash tools/create_data.sh <job_name> sunrgbd
 ```
 
-The above point cloud data are further saved in `.bin` format. Meanwhile `.pkl` info files are also generated for saving annotation and metadata. The core function `process_single_scene` of getting data infos is as follows.
-
-```python
-def process_single_scene(sample_idx):
-    print(f'{self.split} sample_idx: {sample_idx}')
-    # convert depth to points
-    # and downsample the points
-    SAMPLE_NUM = 50000
-    pc_upright_depth = self.get_depth(sample_idx)
-    pc_upright_depth_subsampled = random_sampling(
-        pc_upright_depth, SAMPLE_NUM)
-
-    info = dict()
-    pc_info = {'num_features': 6, 'lidar_idx': sample_idx}
-    info['point_cloud'] = pc_info
-
-    # save point cloud data in `.bin` format
-    mmengine.mkdir_or_exist(osp.join(self.root_dir, 'points'))
-    pc_upright_depth_subsampled.tofile(
-        osp.join(self.root_dir, 'points', f'{sample_idx:06d}.bin'))
-
-    # save point cloud file path
-    info['pts_path'] = osp.join('points', f'{sample_idx:06d}.bin')
-
-    # save image file path and metainfo
-    img_path = osp.join('image', f'{sample_idx:06d}.jpg')
-    image_info = {
-        'image_idx': sample_idx,
-        'image_shape': self.get_image_shape(sample_idx),
-        'image_path': img_path
-    }
-    info['image'] = image_info
-
-    # save calibration information
-    K, Rt = self.get_calibration(sample_idx)
-    calib_info = {'K': K, 'Rt': Rt}
-    info['calib'] = calib_info
-
-    # save all annotation
-    if has_label:
-        obj_list = self.get_label_objects(sample_idx)
-        annotations = {}
-        annotations['gt_num'] = len([
-            obj.classname for obj in obj_list
-            if obj.classname in self.cat2label.keys()
-        ])
-        if annotations['gt_num'] != 0:
-            # class name
-            annotations['name'] = np.array([
-                obj.classname for obj in obj_list
-                if obj.classname in self.cat2label.keys()
-            ])
-            # 2D image bounding boxes
-            annotations['bbox'] = np.concatenate([
-                obj.box2d.reshape(1, 4) for obj in obj_list
-                if obj.classname in self.cat2label.keys()
-            ], axis=0)
-            # 3D bounding box center location (in depth coordinate system)
-            annotations['location'] = np.concatenate([
-                obj.centroid.reshape(1, 3) for obj in obj_list
-                if obj.classname in self.cat2label.keys()
-            ], axis=0)
-            # 3D bounding box dimension/size (in depth coordinate system)
-            annotations['dimensions'] = 2 * np.array([
-                [obj.l, obj.h, obj.w] for obj in obj_list
-                if obj.classname in self.cat2label.keys()
-            ])
-            # 3D bounding box rotation angle/yaw angle (in depth coordinate system)
-            annotations['rotation_y'] = np.array([
-                obj.heading_angle for obj in obj_list
-                if obj.classname in self.cat2label.keys()
-            ])
-            annotations['index'] = np.arange(
-                len(obj_list), dtype=np.int32)
-            # class label (number)
-            annotations['class'] = np.array([
-                self.cat2label[obj.classname] for obj in obj_list
-                if obj.classname in self.cat2label.keys()
-            ])
-            # 3D bounding box (in depth coordinate system)
-            annotations['gt_boxes_upright_depth'] = np.stack(
-                [
-                    obj.box3d for obj in obj_list
-                    if obj.classname in self.cat2label.keys()
-                ], axis=0)  # (K,8)
-        info['annos'] = annotations
-    return info
-```
+The above point cloud data are further saved in `.bin` format. Meanwhile `.pkl` info files are also generated for saving annotation and metadata.
 
 The directory structure after processing should be as follows.
 
@@ -240,22 +153,19 @@ sunrgbd
 
 - `points/0xxxxx.bin`: The point cloud data after downsample.
 - `sunrgbd_infos_train.pkl`: The train data infos, the detailed info of each scene is as follows:
-  - info\['point_cloud'\]: `·`{'num_features': 6, 'lidar_idx': sample_idx}`, where `sample_idx\` is the index of the scene.
-  - info\['pts_path'\]: The path of `points/0xxxxx.bin`.
-  - info\['image'\]: The image path and metainfo:
-    - image\['image_idx'\]: The index of the image.
-    - image\['image_shape'\]: The shape of the image tensor.
-    - image\['image_path'\]: The path of the image.
-  - info\['annos'\]: The annotations of each scene.
-    - annotations\['gt_num'\]: The number of ground truths.
-    - annotations\['name'\]: The semantic name of all ground truths, e.g. `chair`.
-    - annotations\['location'\]: The gravity center of the 3D bounding boxes in depth coordinate system. Shape: \[K, 3\], K is the number of ground truths.
-    - annotations\['dimensions'\]: The dimensions of the 3D bounding boxes in depth coordinate system, i.e. `(x_size, y_size, z_size)`, shape: \[K, 3\].
-    - annotations\['rotation_y'\]: The yaw angle of the 3D bounding boxes in depth coordinate system. Shape: \[K, \].
-    - annotations\['gt_boxes_upright_depth'\]: The 3D bounding boxes in depth coordinate system, each bounding box is `(x, y, z, x_size, y_size, z_size, yaw)`, shape: \[K, 7\].
-    - annotations\['bbox'\]: The 2D bounding boxes, each bounding box is `(x, y, x_size, y_size)`, shape: \[K, 4\].
-    - annotations\['index'\]: The index of all ground truths, range \[0, K).
-    - annotations\['class'\]: The train class id of the bounding boxes, value range: \[0, 10), shape: \[K, \].
+  - info\['lidar_points'\]: A dict contains all information relate to the the lidar points.
+    - info\['lidar_points'\]\['num_pts_feats'\]: The feature dimension of point.
+    - info\['lidar_points'\]\['lidar_path'\]: The filename of `xxx.bin` of lidar points.
+  - info\['images'\]: A dict contains all information relate to the image data.
+    - info\['images'\]\['CAM0'\]\['img_path'\]: The image file name.
+    - info\['images'\]\['CAM0'\]\['depth2img'\]: Transformation matrix from depth to image with shape (4, 4).
+    - info\['images'\]\['CAM0'\]\['height'\]: The height of image.
+    - info\['images'\]\['CAM0'\]\['width'\]: The width of image.
+  - info\['instances'\]: A list of dict contains all the annotations of this frame. Each dict corresponds to annotations of single instance.
+    - info\['instances'\]\['bbox_3d'\]: List of 7 numbers representing the 3D bounding box in depth coordinate system.
+    - info\['instances'\]\['bbox'\]: List of 4 numbers representing the 2D bounding box of the instance, in (x1, y1, x2, y2) order.
+    - info\['instances'\]\['bbox_label_3d'\]: An int indicates the 3D label of instance and the -1 indicates ignore class.
+    - info\['instances'\]\['bbox_label'\]: An int indicates the 2D label of instance and the -1 indicates ignore class.
 - `sunrgbd_infos_val.pkl`: The val data infos, which shares the same format as `sunrgbd_infos_train.pkl`.
 
 ## Train pipeline
@@ -282,8 +192,9 @@ train_pipeline = [
         scale_ratio_range=[0.85, 1.15],
         shift_height=True),
     dict(type='PointSample', num_points=20000),
-    dict(type='DefaultFormatBundle3D', class_names=class_names),
-    dict(type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d'])
+    dict(
+        type='Pack3DDetInputs',
+        keys=['points', 'gt_bboxes_3d', 'gt_labels_3d'])
 ]
 ```
 
@@ -306,9 +217,8 @@ train_pipeline = [
     dict(type='LoadImageFromFile'),
     dict(type='LoadAnnotations3D'),
     dict(type='LoadAnnotations', with_bbox=True),
-    dict(type='Resize', img_scale=(1333, 600), keep_ratio=True),
+    dict(type='Resize', scale=(1333, 600), keep_ratio=True),
     dict(type='RandomFlip', flip_ratio=0.0),
-    dict(type='Normalize', **img_norm_cfg),
     dict(type='Pad', size_divisor=32),
     dict(
         type='RandomFlip3D',
@@ -320,28 +230,21 @@ train_pipeline = [
         rot_range=[-0.523599, 0.523599],
         scale_ratio_range=[0.85, 1.15],
         shift_height=True),
-    dict(type='PointSample', num_points=20000),
-    dict(type='DefaultFormatBundle3D', class_names=class_names),
     dict(
-        type='Collect3D',
-        keys=[
-            'img', 'gt_bboxes', 'gt_labels', 'points', 'gt_bboxes_3d',
-            'gt_labels_3d'
-        ])
+        type='Pack3DDetInputs',
+        keys=['points', 'gt_bboxes_3d', 'gt_labels_3d','img', 'gt_bboxes', 'gt_bboxes_labels', ])
 ]
 ```
 
 Data augmentation/normalization for images:
 
 - `Resize`: resize the input image, `keep_ratio=True` means the ratio of the image is kept unchanged.
-- `Normalize`: normalize the RGB channels of the input image.
 - `RandomFlip`: randomly flip the input image.
-- `Pad`: pad the input image with zeros by default.
 
 The image augmentation and normalization functions are implemented in [MMDetection](https://github.com/open-mmlab/mmdetection/tree/master/mmdet/datasets/pipelines).
 
 ## Metrics
 
-Same as ScanNet, typically mean Average Precision (mAP) is used for evaluation on SUN RGB-D, e.g. `mAP@0.25` and `mAP@0.5`. In detail, a generic function to compute precision and recall for 3D object detection for multiple classes is called, please refer to [indoor_eval](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/evaluation/indoor_eval.py).
+Same as ScanNet, typically mean Average Precision (mAP) is used for evaluation on SUN RGB-D, e.g. `mAP@0.25` and `mAP@0.5`. In detail, a generic function to compute precision and recall for 3D object detection for multiple classes is called, please refer to [indoor_eval](https://github.com/open-mmlab/mmdetection3d/blob/dev-1.x/mmdet3d/evaluation/functional/indoor_eval.py).
 
 Since SUN RGB-D consists of image data, detection on image data is also feasible. For instance, in ImVoteNet, we first train an image detector, and we also use mAP for evaluation, e.g. `mAP@0.5`. We use the `eval_map` function from [MMDetection](https://github.com/open-mmlab/mmdetection) to calculate mAP.

docs/en/getting_started.md

@@ -18,6 +18,8 @@ If you are experienced with PyTorch and have already installed it, just skip thi
 **Step 1.** Create a conda environment and activate it.
 
 ```shell
+# We recommend to install python=3.8 since the waymo-open-dataset-tf-2-6-0 requires python>=3.7
+# If you want to install python<3.7, make sure to install waymo-open-dataset-tf-2-x-0 (x<=4)
 conda create --name openmmlab python=3.8 -y
 conda activate openmmlab
 ```
@@ -254,6 +256,8 @@ docker run --gpus all --shm-size=8g -it -v {DATA_DIR}:/mmdetection3d/data mmdete
 Here is a full script for setting up MMdetection3D with conda.
 
 ```shell
+# We recommend to install python=3.8 since the waymo-open-dataset-tf-2-6-0 requires python>=3.7
+# If you want to install python<3.7, make sure to install waymo-open-dataset-tf-2-x-0 (x<=4)
 conda create -n open-mmlab python=3.8 -y
 conda activate open-mmlab
 

docs/en/notes/changelog.md

 # Changelog of v1.1
 
+### v1.1.0rc1 (11/10/2022)
+
+#### Highlights
+
+- Support a camera-only 3D detection baseline on Waymo, [MV-FCOS3D++](https://arxiv.org/abs/2207.12716)
+
+#### New Features
+
+- Support a camera-only 3D detection baseline on Waymo, [MV-FCOS3D++](https://arxiv.org/abs/2207.12716), with new evaluation metrics and transformations (#1716)
+- Refactor PointRCNN in the framework of mmdet3d v1.1 (#1819)
+
+#### Improvements
+
+- Add `auto_scale_lr` in config to support training with auto-scale learning rates (#1807)
+- Fix CI (#1813, #1865, #1877)
+- Update `browse_dataset.py` script (#1817)
+- Update SUN RGB-D and Lyft datasets documentation (#1833)
+- Rename `convert_to_datasample` to `add_pred_to_datasample` in detectors (#1843)
+- Update customized dataset documentation (#1845)
+- Update `Det3DLocalVisualization` and visualization documentation (#1857)
+- Add the code of generating `cam_sync_labels` for Waymo dataset (#1870)
+- Update dataset transforms typehints (#1875)
+
+#### Bug Fixes
+
+- Fix missing registration of models in [setup_env.py](https://github.com/open-mmlab/mmdetection3d/blob/dev-1.x/mmdet3d/utils/setup_env.py) (#1808)
+- Fix the data base sampler bugs when using the ground plane data (#1812)
+- Add output directory existing check during visualization (#1828)
+- Fix bugs of nuScenes dataset for monocular 3D detection (#1837)
+- Fix visualization hook to support the visualization of different data modalities (#1839)
+- Fix monocular 3D detection demo (#1864)
+- Fix the lack of `num_pts_feats` key in nuscenes dataset and complete docstring (#1882)
+
+#### Contributors
+
+A total of 10 developers contributed to this release.
+
+@ZwwWayne, @Tai-Wang, @lianqing11, @VVsssssk, @ZCMax, @Xiangxu-0103, @JingweiZhang12, @tpoisonooo, @ice-tong, @jshilong
+
 ### v1.1.0rc0 (1/9/2022)
 
 We are excited to announce the release of MMDetection3D 1.1.0rc0.

docs/en/notes/changelog_v1.0.x.md

 # Changelog of v1.0.x
 
+### v1.0.0rc5 (11/10/2022)
+
+#### New Features
+
+- Support ImVoxelNet on SUN RGB-D (#1738)
+
+#### Improvements
+
+- Fix the cross-codebase reference problem in metafile README (#1644)
+- Update the Chinese documentation about getting started (#1715)
+- Fix docs link and add docs link checker (#1811)
+
+#### Bug Fixes
+
+- Fix a visualization bug that is potentially triggered by empty prediction labels (#1725)
+- Fix point cloud segmentation visualization bug due to wrong parameter passing (#1858)
+- Fix Nan loss bug during PointRCNN training (#1874)
+
+#### Contributors
+
+A total of 9 developers contributed to this release.
+
+@ZwwWayne, @Tai-Wang, @filaPro, @VVsssssk, @ZCMax, @Xiangxu-0103, @holtvogt, @tpoisonooo, @lianqing01
+
 ### v1.0.0rc4 (8/8/2022)
 
 #### Highlights

docs/en/notes/faq.md

@@ -6,14 +6,15 @@ We list some potential troubles encountered by users and developers, along with
 
 - Compatibility issue between MMEngine, MMCV, MMDetection and MMDetection3D; "ConvWS is already registered in conv layer"; "AssertionError: MMCV==xxx is used but incompatible. Please install mmcv>=xxx, \<=xxx."
 
-  The required versions of MMEngine, MMCV and MMDetection for different versions of MMDetection3D are as below. Please install the correct version of MMEngine, MMCV and MMDetection to avoid installation issues.
+- The required versions of MMEngine, MMCV and MMDetection for different versions of MMDetection3D are as below. Please install the correct version of MMEngine, MMCV and MMDetection to avoid installation issues.
 
-  | MMDetection3D version |     MMEngine version     |      MMCV version       |   MMDetection version    |
-  | :-------------------: | :----------------------: | :---------------------: | :----------------------: |
-  |        dev-1.x        | mmengine>=0.1.0, \<0.2.0 | mmcv>=2.0.0rc0, \<2.1.0 | mmdet>=3.0.0rc0, \<3.1.0 |
-  |       v1.1.0rc0       | mmengine>=0.1.0, \<0.2.0 | mmcv>=2.0.0rc0, \<2.1.0 | mmdet>=3.0.0rc0, \<3.1.0 |
+| MMDetection3D version |     MMEngine version     |      MMCV version       |   MMDetection version    |
+| --------------------- | :----------------------: | :---------------------: | :----------------------: |
+| dev-1.x               | mmengine>=0.1.0, \<0.2.0 | mmcv>=2.0.0rc0, \<2.1.0 | mmdet>=3.0.0rc0, \<3.1.0 |
+| v1.1.0rc1             | mmengine>=0.1.0, \<0.2.0 | mmcv>=2.0.0rc0, \<2.1.0 | mmdet>=3.0.0rc0, \<3.1.0 |
+| v1.1.0rc0             | mmengine>=0.1.0, \<0.2.0 | mmcv>=2.0.0rc0, \<2.1.0 | mmdet>=3.0.0rc0, \<3.1.0 |
 
-  **Note:** If you want to install mmdet3d-v1.0.0x, the compatible MMDetection, MMSegmentation and MMCV versions table can be found at [here](https://mmdetection3d.readthedocs.io/en/latest/faq.html#mmcv-mmdet-mmdet3d-installation). Please choose the correct version of MMCV to avoid installation issues.
+**Note:** If you want to install mmdet3d-v1.0.0x, the compatible MMDetection, MMSegmentation and MMCV versions table can be found at [here](https://mmdetection3d.readthedocs.io/en/latest/faq.html#mmcv-mmdet-mmdet3d-installation). Please choose the correct version of MMCV to avoid installation issues.
 
 - If you faced the error shown below when importing open3d:
 

docs/en/user_guides/config.md

@@ -411,7 +411,7 @@ param_scheduler = [
 
 ### Hook config
 
-Users can attach hooks to training, validation, and testing loops to insert some oprations during running. There are two different hook fields, one is `default_hooks` and the other is `custom_hooks`.
+Users can attach hooks to training, validation, and testing loops to insert some operations during running. There are two different hook fields, one is `default_hooks` and the other is `custom_hooks`.
 
 `default_hooks` is a dict of hook configs. `default_hooks` are the hooks must required at runtime. They have default priority which should not be modified. If not set, runner will use the default values. To disable a default hook, users can set its config to `None`.
 

docs/en/user_guides/index.rst

@@ -11,3 +11,4 @@
    index.rst
    model_deployment.md
    useful_tools.md
+   visualization.md

docs/en/user_guides/visualization.md

+# Visualization
+
+MMDetection3D provides a `Det3DLocalVisualizer` to visualize and store the state of the model during training and testing, as well as results, with the following features.
+
+1. Support the basic drawing interface for multi-modality data and multi-task.
+2. Support multiple backends such as local, TensorBoard, to write training status such as `loss`, `lr`, or performance evaluation metrics and to a specified single or multiple backends.
+3. Support ground truth visualization on multimodal data, and cross-modal visualization of 3D detection results.
+
+## Basic Drawing Interface
+
+Inherited from `DetLocalVisualizer`, `Det3DLocalVisualizer` provides an interface for drawing common objects on 2D images, such as drawing detection boxes, points, text, lines, circles, polygons, and binary masks. More details about 2D drawing can refer to the [visualization documentation](https://mmengine.readthedocs.io/zh_CN/latest/advanced_tutorials/visualization.html) in MMDetection. Here we introduce the 3D drawing interface:
+
+### Drawing point cloud on the image
+
+We support drawing point cloud on the image by using `draw_points_on_image`.
+
+```python
+import mmcv
+from mmengine import load
+from mmdet3d.visualization import Det3DLocalVisualizer
+import numpy as np
+info_file = load('demo/data/kitti/000008.pkl')
+points = np.fromfile('demo/data/kitti/000008.bin', dtype=np.float32)
+points = points.reshape(-1, 4)[:, :3]
+lidar2img = np.array(info_file['data_list'][0]['images']['CAM2']['lidar2img'], dtype=np.float32)
+
+visualizer = Det3DLocalVisualizer()
+img = mmcv.imread('demo/data/kitti/000008.png')
+img = mmcv.imconvert(img, 'bgr', 'rgb')
+visualizer.set_image(img)
+visualizer.draw_points_on_image(points, lidar2img)
+visualizer.show()
+```
+
+![points_on_image](../../../resources/points_on_image.png)
+
+### Drawing 3D Boxes on Point Cloud
+
+We support drawing 3D boxes on point cloud by using `draw_bboxes_3d`.
+
+```python
+import torch
+from mmdet3d.visualization import Det3DLocalVisualizer
+from mmdet3d.structures import LiDARInstance3DBoxes
+
+points = np.fromfile('tests/data/kitti/training/velodyne/000000.bin', dtype=np.float32)
+points = points.reshape(-1, 4)
+visualizer = Det3DLocalVisualizer()
+# set point cloud in visualizer
+visualizer.set_points(points)
+bboxes_3d = LiDARInstance3DBoxes(torch.tensor(
+                [[8.7314, -1.8559, -1.5997, 1.2000, 0.4800, 1.8900,
+                  -1.5808]])),
+# Draw 3D bboxes
+visualizer.draw_bboxes_3d(bboxes_3d)
+visualizer.show()
+```
+
+![mono3d](../../../resources/pcd.png)
+
+### Drawing Projected 3D Boxes on Image
+
+We support drawing projected 3D boxes on image by using `draw_proj_bboxes_3d`.
+
+```python
+import mmcv
+import numpy as np
+from mmengine import load
+from mmdet3d.visualization import Det3DLocalVisualizer
+from mmdet3d.structures import CameraInstance3DBoxes
+info_file = load('demo/data/kitti/000008.pkl')
+cam2img = np.array(info_file['data_list'][0]['images']['CAM2']['cam2img'], dtype=np.float32)
+bboxes_3d = []
+for instance in info_file['data_list'][0]['instances']:
+    bboxes_3d.append(instance['bbox_3d'])
+gt_bboxes_3d = np.array(bboxes_3d, dtype=np.float32)
+gt_bboxes_3d = CameraInstance3DBoxes(gt_bboxes_3d)
+input_meta = {'cam2img': cam2img}
+
+visualizer = Det3DLocalVisualizer()
+
+img = mmcv.imread('demo/data/kitti/000008.png')
+img = mmcv.imconvert(img, 'bgr', 'rgb')
+visualizer.set_image(img)
+# project 3D bboxes to image
+visualizer.draw_proj_bboxes_3d(gt_bboxes_3d, input_meta)
+visualizer.show()
+```
+
+![mono3d](../../../resources/mono3d.png)
+
+### Drawing BEV Boxes
+
+We support drawing projected 3D boxes on image by using `draw_proj_bboxes_3d`.
+
+```python
+import numpy as np
+from mmengine import load
+from mmdet3d.visualization import Det3DLocalVisualizer
+from mmdet3d.structures import CameraInstance3DBoxes
+
+info_file = load('demo/data/kitti/000008.pkl')
+bboxes_3d = []
+for instance in info_file['data_list'][0]['instances']:
+    bboxes_3d.append(instance['bbox_3d'])
+gt_bboxes_3d = np.array(bboxes_3d, dtype=np.float32)
+gt_bboxes_3d = CameraInstance3DBoxes(gt_bboxes_3d)
+
+visualizer = Det3DLocalVisualizer()
+# set bev image in visualizer
+visualizer.set_bev_image()
+# draw bev bboxes
+visualizer.draw_bev_bboxes(gt_bboxes_3d, edge_colors='orange')
+visualizer.show()
+```
+
+<img src="../../../resources/bev.png" width = "50%" />
+
+### Drawing 3D Semantic Mask
+
+We support draw segmentation mask via per-point colorization by using `draw_seg_mask`.
+
+```python
+import torch
+from mmdet3d.visualization import Det3DLocalVisualizer
+
+points = np.fromfile('tests/data/s3dis/points/Area_1_office_2.bin', dtype=np.float32)
+points = points.reshape(-1, 3)
+visualizer = Det3DLocalVisualizer()
+mask = np.random.rand(points.shape[0], 3)
+points_with_mask = np.concatenate((points, mask), axis=-1)
+# Draw 3D points with mask
+visualizer.draw_seg_mask(points_with_mask)
+visualizer.show()
+```
+
+## Results
+
+To see the prediction results of trained models, you can run the following command
+
+```bash
+python tools/test.py ${CONFIG_FILE} ${CKPT_PATH} --show --show-dir ${SHOW_DIR}
+```
+
+After running this command, plotted results including input data and the output of networks visualized on the input will be saved in `${SHOW_DIR}`.
+
+After running this command, you will obtain the input data, the output of networks and ground-truth labels visualized on the input (e.g. `***_gt.png` and `***_pred.png` in multi-modality detection task and vision-based detection task) in `${SHOW_DIR}`. When `show` is enabled, [Open3D](http://www.open3d.org/) will be used to visualize the results online. If you are running test in remote server without GUI, the online visualization is not supported, you can download the `results.pkl` from the remote server, and visualize the prediction results offline in your local machine.
+
+To visualize the results with `Open3D` backend offline, you can run the following command
+
+```bash
+python tools/misc/visualize_results.py ${CONFIG_FILE} --result ${RESULTS_PATH} --show-dir ${SHOW_DIR}
+```
+
+![](../../../resources/open3d_visual.gif)
+
+This allows the inference and results generation to be done in remote server and the users can open them on their host with GUI.
+
+## Dataset
+
+We also provide scripts to visualize the dataset without inference. You can use `tools/misc/browse_dataset.py` to show loaded data and ground-truth online and save them on the disk. Currently we support single-modality 3D detection and 3D segmentation on all the datasets, multi-modality 3D detection on KITTI and SUN RGB-D, as well as monocular 3D detection on nuScenes. To browse the KITTI dataset, you can run the following command
+
+```shell
+python tools/misc/browse_dataset.py configs/_base_/datasets/kitti-3d-3class.py --task det --output-dir ${OUTPUT_DIR}
+```
+
+**Notice**: Once specifying `--output-dir`, the images of views specified by users will be saved when pressing `_ESC_` in open3d window.
+
+To verify the data consistency and the effect of data augmentation, you can also add `--aug` flag to visualize the data after data augmentation using the command as below:
+
+```shell
+python tools/misc/browse_dataset.py configs/_base_/datasets/kitti-3d-3class.py --task det --aug --output-dir ${OUTPUT_DIR}
+```
+
+If you also want to show 2D images with 3D bounding boxes projected onto them, you need to find a config that supports multi-modality data loading, and then change the `--task` args to `multi_modality-det`. An example is showed below
+
+```shell
+python tools/misc/browse_dataset.py configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py --task multi-modality_det --output-dir ${OUTPUT_DIR}
+```
+
+![](../../../resources/browse_dataset_multi_modality.png)
+
+You can simply browse different datasets using different configs, e.g. visualizing the ScanNet dataset in 3D semantic segmentation task
+
+```shell
+python tools/misc/browse_dataset.py configs/_base_/datasets/scannet_seg-3d-20class.py --task seg --output-dir ${OUTPUT_DIR} --online
+```
+
+![](../../../resources/browse_dataset_seg.png)
+
+And browsing the nuScenes dataset in monocular 3D detection task
+
+```shell
+python tools/misc/browse_dataset.py configs/_base_/datasets/nus-mono3d.py --task mono_det --output-dir ${OUTPUT_DIR} --online
+```
+
+![](../../../resources/browse_dataset_mono.png)

docs/zh_cn/getting_started.md

@@ -17,6 +17,8 @@ MMDetection3D 可以安装在 Linux, MacOS, (实验性支持 Windows) 的平台
 **步骤 1.** 使用 conda 新建虚拟环境，并进入该虚拟环境.
 
 ```shell
+# 鉴于 waymo-open-dataset-tf-2-6-0 要求 python>=3.7, 我们推荐安装 python3.8
+# 如果您想要安装 python3.6, 之后须确保安装 waymo-open-dataset-tf-2-x-0 (x<=4)
 conda create --name openmmlab python=3.8 -y
 conda activate openmmlab
 ```
@@ -269,7 +271,9 @@ docker run --gpus all --shm-size=8g -it -v {DATA_DIR}:/mmdetection3d/data mmdete
 以下是一个基于 conda 安装 MMdetection3D 的脚本
 
 ```shell
-conda create -n open-mmlab python=3.7 -y
+# 鉴于 waymo-open-dataset-tf-2-6-0 要求 python>=3.7, 我们推荐安装 python3.8
+# 如果您想要安装 python3.6, 之后须确保安装 waymo-open-dataset-tf-2-x-0 (x<=4)
+conda create -n open-mmlab python=3.8 -y
 conda activate open-mmlab
 
 # install latest PyTorch prebuilt with the default prebuilt CUDA version (usually the latest)

docs/zh_cn/user_guides/demo.md

@@ -67,7 +67,7 @@ python demo/mono_det_demo.py ${IMAGE_FILE} ${ANNOTATION_FILE} ${CONFIG_FILE} ${C
 在 nuScenes 数据上测试 [FCOS3D](https://github.com/open-mmlab/mmdetection3d/tree/master/configs/fcos3d) 模型：
 
 ```shell
-python demo/mono_det_demo.py demo/data/nuscenes/n015-2018-07-24-11-22-45+0800__CAM_BACK__1532402927637525.jpg demo/data/nuscenes/n015-2018-07-24-11-22-45+0800__CAM_BACK__1532402927637525_mono3d.coco.json configs/fcos3d/fcos3d_r101-caffe-dcn_fpn_head-gn_8xb2-1x_nus-mono3d_finetune.py checkpoints/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d_finetune_20210717_095645-8d806dc2.pth
+python demo/mono_det_demo.py demo/data/nuscenes/n015-2018-07-24-11-22-45+0800__CAM_BACK__1532402927637525.jpg demo/data/nuscenes/n015-2018-07-24-11-22-45+0800__CAM_BACK__1532402927637525_mono3d.pkl configs/fcos3d/fcos3d_r101-caffe-dcn_fpn_head-gn_8xb2-1x_nus-mono3d_finetune.py checkpoints/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d_finetune_20210717_095645-8d806dc2.pth
 ```
 
 ### 3D 分割