Demo

configs/_base_/datasets/kitti-3d-3class.py

@@ -93,7 +93,10 @@ data = dict(
             pipeline=train_pipeline,
             modality=input_modality,
             classes=class_names,
-            test_mode=False)),
+            test_mode=False,
+            # we use box_type_3d='LiDAR' in kitti and nuscenes dataset
+            # and box_type_3d='Depth' in sunrgbd and scannet dataset.
+            box_type_3d='LiDAR')),
     val=dict(
         type=dataset_type,
         data_root=data_root,
@@ -103,7 +106,8 @@ data = dict(
         pipeline=test_pipeline,
         modality=input_modality,
         classes=class_names,
-        test_mode=True),
+        test_mode=True,
+        box_type_3d='LiDAR'),
     test=dict(
         type=dataset_type,
         data_root=data_root,
@@ -113,6 +117,7 @@ data = dict(
         pipeline=test_pipeline,
         modality=input_modality,
         classes=class_names,
-        test_mode=True))
+        test_mode=True,
+        box_type_3d='LiDAR'))
 
 evaluation = dict(interval=1)

configs/_base_/datasets/kitti-3d-car.py

@@ -91,7 +91,10 @@ data = dict(
             pipeline=train_pipeline,
             modality=input_modality,
             classes=class_names,
-            test_mode=False)),
+            test_mode=False,
+            # we use box_type_3d='LiDAR' in kitti and nuscenes dataset
+            # and box_type_3d='Depth' in sunrgbd and scannet dataset.
+            box_type_3d='LiDAR')),
     val=dict(
         type=dataset_type,
         data_root=data_root,
@@ -101,7 +104,8 @@ data = dict(
         pipeline=test_pipeline,
         modality=input_modality,
         classes=class_names,
-        test_mode=True),
+        test_mode=True,
+        box_type_3d='LiDAR'),
     test=dict(
         type=dataset_type,
         data_root=data_root,
@@ -111,6 +115,7 @@ data = dict(
         pipeline=test_pipeline,
         modality=input_modality,
         classes=class_names,
-        test_mode=True))
+        test_mode=True,
+        box_type_3d='LiDAR'))
 
 evaluation = dict(interval=1)

configs/_base_/datasets/nus-3d.py

@@ -92,7 +92,10 @@ data = dict(
         pipeline=train_pipeline,
         classes=class_names,
         modality=input_modality,
-        test_mode=False),
+        test_mode=False,
+        # we use box_type_3d='LiDAR' in kitti and nuscenes dataset
+        # and box_type_3d='Depth' in sunrgbd and scannet dataset.
+        box_type_3d='LiDAR'),
     val=dict(
         type=dataset_type,
         data_root=data_root,
@@ -100,7 +103,8 @@ data = dict(
         pipeline=test_pipeline,
         classes=class_names,
         modality=input_modality,
-        test_mode=True),
+        test_mode=True,
+        box_type_3d='LiDAR'),
     test=dict(
         type=dataset_type,
         data_root=data_root,
@@ -108,7 +112,8 @@ data = dict(
         pipeline=test_pipeline,
         classes=class_names,
         modality=input_modality,
-        test_mode=True))
+        test_mode=True,
+        box_type_3d='LiDAR'))
 # For nuScenes dataset, we usually evaluate the model at the end of training.
 # Since the models are trained by 24 epochs by default, we set evaluation
 # interval to be 24. Please change the interval accordingly if you do not

configs/_base_/datasets/scannet-3d-18class.py

@@ -83,18 +83,23 @@ data = dict(
             ann_file=data_root + 'scannet_infos_train.pkl',
             pipeline=train_pipeline,
             filter_empty_gt=False,
-            classes=class_names)),
+            classes=class_names,
+            # we use box_type_3d='LiDAR' in kitti and nuscenes dataset
+            # and box_type_3d='Depth' in sunrgbd and scannet dataset.
+            box_type_3d='Depth')),
     val=dict(
         type=dataset_type,
         data_root=data_root,
         ann_file=data_root + 'scannet_infos_val.pkl',
         pipeline=test_pipeline,
         classes=class_names,
-        test_mode=True),
+        test_mode=True,
+        box_type_3d='Depth'),
     test=dict(
         type=dataset_type,
         data_root=data_root,
         ann_file=data_root + 'scannet_infos_val.pkl',
         pipeline=test_pipeline,
         classes=class_names,
-        test_mode=True))
+        test_mode=True,
+        box_type_3d='Depth'))

configs/_base_/datasets/sunrgbd-3d-10class.py

@@ -66,18 +66,23 @@ data = dict(
             ann_file=data_root + 'sunrgbd_infos_train.pkl',
             pipeline=train_pipeline,
             classes=class_names,
-            filter_empty_gt=False)),
+            filter_empty_gt=False,
+            # we use box_type_3d='LiDAR' in kitti and nuscenes dataset
+            # and box_type_3d='Depth' in sunrgbd and scannet dataset.
+            box_type_3d='Depth')),
     val=dict(
         type=dataset_type,
         data_root=data_root,
         ann_file=data_root + 'sunrgbd_infos_val.pkl',
         pipeline=test_pipeline,
         classes=class_names,
-        test_mode=True),
+        test_mode=True,
+        box_type_3d='Depth'),
     test=dict(
         type=dataset_type,
         data_root=data_root,
         ann_file=data_root + 'sunrgbd_infos_val.pkl',
         pipeline=test_pipeline,
         classes=class_names,
-        test_mode=True))
+        test_mode=True,
+        box_type_3d='Depth'))

demo/inference_demo.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "pycharm": {
+     "is_executing": false
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from mmdet3d.apis import init_detector, inference_detector, show_result_meshlab"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "pycharm": {
+     "is_executing": false
+    }
+   },
+   "outputs": [],
+   "source": [
+    "config_file = '../configs/second/hv_second_secfpn_6x8_80e_kitti-3d-car.py'\n",
+    "# download the checkpoint from model zoo and put it in `checkpoints/`\n",
+    "checkpoint_file = '../work_dirs/second/epoch_40.pth'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "pycharm": {
+     "is_executing": false
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# build the model from a config file and a checkpoint file\n",
+    "model = init_detector(config_file, checkpoint_file, device='cuda:0')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "pycharm": {
+     "is_executing": false
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# test a single sample\n",
+    "pcd = 'kitti_000008.bin'\n",
+    "result, data = inference_detector(model, pcd)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "pycharm": {
+     "is_executing": false
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# show the results\n",
+    "out_dir = './'\n",
+    "show_result_meshlab(data, result, out_dir)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.7"
+  },
+  "pycharm": {
+   "stem_cell": {
+    "cell_type": "raw",
+    "source": [],
+    "metadata": {
+     "collapsed": false
+    }
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

demo/pcd_demo.py

+from argparse import ArgumentParser
+
+from mmdet3d.apis import inference_detector, init_detector, show_result_meshlab
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument('pcd', help='Point cloud file')
+    parser.add_argument('config', help='Config file')
+    parser.add_argument('checkpoint', help='Checkpoint file')
+    parser.add_argument(
+        '--device', default='cuda:0', help='Device used for inference')
+    parser.add_argument(
+        '--score-thr', type=float, default=0.6, help='bbox score threshold')
+    parser.add_argument(
+        '--out-dir', type=str, default='demo', help='dir to save results')
+    args = parser.parse_args()
+
+    # build the model from a config file and a checkpoint file
+    model = init_detector(args.config, args.checkpoint, device=args.device)
+    # test a single image
+    result, data = inference_detector(model, args.pcd)
+    # show the results
+    show_result_meshlab(data, result, args.out_dir)
+
+
+if __name__ == '__main__':
+    main()

docs/getting_started.md

@@ -176,34 +176,19 @@ You can use 3D visualization software such as the [MeshLab](http://www.meshlab.n
 
 **Notice**: The visualization API is a little unstable since we plan to refactor these parts together with MMDetection in the future.
 
-### Image demo
+### Point cloud demo
 
-We provide a demo script to test a single image.
+We provide a demo script to test a single sample.
 
 ```shell
-python demo/image_demo.py ${IMAGE_FILE} ${CONFIG_FILE} ${CHECKPOINT_FILE} [--device ${GPU_ID}] [--score-thr ${SCORE_THR}]
+python demo/pcd_demo.py ${PCD_FILE} ${CONFIG_FILE} ${CHECKPOINT_FILE} [--device ${GPU_ID}] [--score-thr ${SCORE_THR}] [--out-dir ${OUT_DIR}]
 ```
 
 Examples:
 
 ```shell
-python demo/image_demo.py demo/demo.jpg configs/faster_rcnn_r50_fpn_1x_coco.py \
-    checkpoints/faster_rcnn_r50_fpn_1x_20181010-3d1b3351.pth --device cpu
-```
-
-### Webcam demo
-
-We provide a webcam demo to illustrate the results.
-
-```shell
-python demo/webcam_demo.py ${CONFIG_FILE} ${CHECKPOINT_FILE} [--device ${GPU_ID}] [--camera-id ${CAMERA-ID}] [--score-thr ${SCORE_THR}]
-```
-
-Examples:
-
-```shell
-python demo/webcam_demo.py configs/faster_rcnn_r50_fpn_1x_coco.py \
-    checkpoints/faster_rcnn_r50_fpn_1x_20181010-3d1b3351.pth
+python demo/pcd_demo.py demo/kitti_000008.bin configs/second/hv_second_secfpn_6x8_80e_kitti-3d-car.py \
+    checkpoints/epoch_40.pth
 ```
 
 

mmdet3d/apis/__init__.py

+from .inference import inference_detector, init_detector, show_result_meshlab
 from .test import single_gpu_test
 
-__all__ = ['single_gpu_test']
+__all__ = [
+    'inference_detector', 'init_detector', 'single_gpu_test',
+    'show_result_meshlab'
+]

mmdet3d/apis/inference.py

+import os.path as osp
+from copy import deepcopy
+
+import mmcv
+import torch
+from mmcv.parallel import collate, scatter
+from mmcv.runner import load_checkpoint
+
+from mmdet3d.core import Box3DMode, show_result
+from mmdet3d.core.bbox import get_box_type
+from mmdet3d.datasets.pipelines import Compose
+from mmdet3d.models import build_detector
+
+
+def init_detector(config, checkpoint=None, device='cuda:0'):
+    """Initialize a detector from config file.
+
+    Args:
+        config (str or :obj:`mmcv.Config`): Config file path or the config
+            object.
+        checkpoint (str, optional): Checkpoint path. If left as None, the model
+            will not load any weights.
+        device (str): Device to use.
+
+    Returns:
+        nn.Module: The constructed detector.
+    """
+    if isinstance(config, str):
+        config = mmcv.Config.fromfile(config)
+    elif not isinstance(config, mmcv.Config):
+        raise TypeError('config must be a filename or Config object, '
+                        f'but got {type(config)}')
+    config.model.pretrained = None
+    model = build_detector(config.model, test_cfg=config.test_cfg)
+    if checkpoint is not None:
+        checkpoint = load_checkpoint(model, checkpoint)
+        if 'CLASSES' in checkpoint['meta']:
+            model.CLASSES = checkpoint['meta']['CLASSES']
+        else:
+            model.CLASSES = config.class_names
+    model.cfg = config  # save the config in the model for convenience
+    model.to(device)
+    model.eval()
+    return model
+
+
+def inference_detector(model, pcd):
+    """Inference point cloud with the detector.
+
+    Args:
+        model (nn.Module): The loaded detector.
+        pcd (str): Point cloud files.
+
+    Returns:
+        tuple: Predicted results and data from pipeline.
+    """
+    cfg = model.cfg
+    device = next(model.parameters()).device  # model device
+    # build the data pipeline
+    test_pipeline = deepcopy(cfg.data.test.pipeline)
+    test_pipeline = Compose(test_pipeline)
+    box_type_3d, box_mode_3d = get_box_type(cfg.data.test.box_type_3d)
+    data = dict(
+        pts_filename=pcd,
+        box_type_3d=box_type_3d,
+        box_mode_3d=box_mode_3d,
+        img_fields=[],
+        bbox3d_fields=[],
+        pts_mask_fields=[],
+        pts_seg_fields=[],
+        bbox_fields=[],
+        mask_fields=[],
+        seg_fields=[])
+    data = test_pipeline(data)
+    data = collate([data], samples_per_gpu=1)
+    if next(model.parameters()).is_cuda:
+        # scatter to specified GPU
+        data = scatter(data, [device.index])[0]
+    else:
+        raise NotImplementedError('Not support cpu-only currently')
+
+    # forward the model
+    with torch.no_grad():
+        result = model(return_loss=False, rescale=True, **data)
+    return result, data
+
+
+def show_result_meshlab(data, result, out_dir):
+    """Show result by meshlab.
+
+    Args:
+        data (dict): Contain data from pipeline.
+        result (dict): Predicted result from model.
+        out_dir (str): Directory to save visualized result.
+    """
+    points = data['points'][0][0].cpu().numpy()
+    pts_filename = data['img_metas'][0][0]['pts_filename']
+    file_name = osp.split(pts_filename)[-1].split('.')[0]
+
+    assert out_dir is not None, 'Expect out_dir, got none.'
+
+    pred_bboxes = result['boxes_3d'].tensor.numpy()
+    # for now we convert points into depth mode
+    if data['img_metas'][0][0]['box_mode_3d'] != Box3DMode.DEPTH:
+        points = points[..., [1, 0, 2]]
+        points[..., 0] *= -1
+        pred_bboxes = Box3DMode.convert(pred_bboxes,
+                                        data['img_metas'][0][0]['box_mode_3d'],
+                                        Box3DMode.DEPTH)
+        pred_bboxes[..., 2] += pred_bboxes[..., 5] / 2
+    else:
+        pred_bboxes[..., 2] += pred_bboxes[..., 5] / 2
+    show_result(points, None, pred_bboxes, out_dir, file_name)

mmdet3d/core/bbox/__init__.py

@@ -8,7 +8,8 @@ from .samplers import (BaseSampler, CombinedSampler,
                        PseudoSampler, RandomSampler, SamplingResult)
 from .structures import (BaseInstance3DBoxes, Box3DMode, CameraInstance3DBoxes,
                          DepthInstance3DBoxes, LiDARInstance3DBoxes,
-                         limit_period, points_cam2img, xywhr2xyxyr)
+                         get_box_type, limit_period, points_cam2img,
+                         xywhr2xyxyr)
 from .transforms import bbox3d2result, bbox3d2roi, bbox3d_mapping_back
 
 from .assign_sampling import (  # isort:skip, avoid recursive imports
@@ -16,7 +17,7 @@ from .assign_sampling import (  # isort:skip, avoid recursive imports
     assign_and_sample, build_assigner, build_sampler)
 
 __all__ = [
-    'BaseAssigner', 'MaxIoUAssigner', 'AssignResult', 'BaseSampler',
+    'BaseSampler', 'AssignResult', 'BaseAssigner', 'MaxIoUAssigner',
     'PseudoSampler', 'RandomSampler', 'InstanceBalancedPosSampler',
     'IoUBalancedNegSampler', 'CombinedSampler', 'SamplingResult',
     'build_assigner', 'build_sampler', 'assign_and_sample', 'build_bbox_coder',
@@ -24,5 +25,6 @@ __all__ = [
     'bbox_overlaps_nearest_3d', 'bbox_overlaps_3d', 'Box3DMode',
     'LiDARInstance3DBoxes', 'CameraInstance3DBoxes', 'bbox3d2roi',
     'bbox3d2result', 'DepthInstance3DBoxes', 'BaseInstance3DBoxes',
-    'bbox3d_mapping_back', 'xywhr2xyxyr', 'limit_period', 'points_cam2img'
+    'bbox3d_mapping_back', 'xywhr2xyxyr', 'limit_period', 'points_cam2img',
+    'get_box_type'
 ]

mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py

 from mmdet.core.bbox import bbox_overlaps
 from mmdet.core.bbox.iou_calculators.builder import IOU_CALCULATORS
-from ..structures import (CameraInstance3DBoxes, DepthInstance3DBoxes,
-                          LiDARInstance3DBoxes)
+from ..structures import get_box_type
 
 
 @IOU_CALCULATORS.register_module()
@@ -84,16 +83,7 @@ def bbox_overlaps_nearest_3d(bboxes1,
     assert bboxes1.size(-1) >= 7
     assert bboxes2.size(-1) >= 7
 
-    if coordinate == 'camera':
-        box_type = CameraInstance3DBoxes
-    elif coordinate == 'lidar':
-        box_type = LiDARInstance3DBoxes
-    elif coordinate == 'depth':
-        box_type = DepthInstance3DBoxes
-    else:
-        raise ValueError(
-            '"coordinate" should be in ["camera", "lidar", "depth"],'
-            f' got invalid {coordinate}')
+    box_type, _ = get_box_type(coordinate)
 
     bboxes1 = box_type(bboxes1, box_dim=bboxes1.shape[-1])
     bboxes2 = box_type(bboxes2, box_dim=bboxes2.shape[-1])
@@ -129,16 +119,8 @@ def bbox_overlaps_3d(bboxes1, bboxes2, mode='iou', coordinate='camera'):
             with shape (M, N) (aligned mode is not supported currently).
     """
     assert bboxes1.size(-1) == bboxes2.size(-1) == 7
-    if coordinate == 'camera':
-        box_type = CameraInstance3DBoxes
-    elif coordinate == 'lidar':
-        box_type = LiDARInstance3DBoxes
-    elif coordinate == 'depth':
-        box_type = DepthInstance3DBoxes
-    else:
-        raise ValueError(
-            '"coordinate" should be in ["camera", "lidar", "depth"],'
-            f' got invalid {coordinate}')
+
+    box_type, _ = get_box_type(coordinate)
 
     bboxes1 = box_type(bboxes1, box_dim=bboxes1.shape[-1])
     bboxes2 = box_type(bboxes2, box_dim=bboxes2.shape[-1])

mmdet3d/core/bbox/structures/__init__.py

@@ -3,11 +3,11 @@ from .box_3d_mode import Box3DMode
 from .cam_box3d import CameraInstance3DBoxes
 from .depth_box3d import DepthInstance3DBoxes
 from .lidar_box3d import LiDARInstance3DBoxes
-from .utils import (limit_period, points_cam2img, rotation_3d_in_axis,
-                    xywhr2xyxyr)
+from .utils import (get_box_type, limit_period, points_cam2img,
+                    rotation_3d_in_axis, xywhr2xyxyr)
 
 __all__ = [
     'Box3DMode', 'BaseInstance3DBoxes', 'LiDARInstance3DBoxes',
     'CameraInstance3DBoxes', 'DepthInstance3DBoxes', 'xywhr2xyxyr',
-    'rotation_3d_in_axis', 'limit_period', 'points_cam2img'
+    'get_box_type', 'rotation_3d_in_axis', 'limit_period', 'points_cam2img'
 ]

mmdet3d/core/bbox/structures/utils.py

@@ -74,6 +74,35 @@ def xywhr2xyxyr(boxes_xywhr):
     return boxes
 
 
+def get_box_type(box_type):
+    """Get the type and mode of box structure.
+
+    Args:
+        box_type (str): Indicate the type of box structure.
+            The valid value are "LiDAR", "Camera", or "Depth".
+
+    Returns:
+        tuple: box type and box mode.
+    """
+    from .box_3d_mode import (LiDARInstance3DBoxes, CameraInstance3DBoxes,
+                              DepthInstance3DBoxes, Box3DMode)
+    box_type_lower = box_type.lower()
+    if box_type_lower == 'lidar':
+        box_type_3d = LiDARInstance3DBoxes
+        box_mode_3d = Box3DMode.LIDAR
+    elif box_type_lower == 'camera':
+        box_type_3d = CameraInstance3DBoxes
+        box_mode_3d = Box3DMode.CAM
+    elif box_type_lower == 'depth':
+        box_type_3d = DepthInstance3DBoxes
+        box_mode_3d = Box3DMode.DEPTH
+    else:
+        raise ValueError('Only "box_type" of "camera", "lidar", "depth"'
+                         f' are supported, got {box_type}')
+
+    return box_type_3d, box_mode_3d
+
+
 def points_cam2img(points_3d, proj_mat):
     points_num = list(points_3d.shape)[:-1]
     points_shape = np.concatenate([points_num, [1]], axis=0).tolist()

mmdet3d/datasets/__init__.py

@@ -14,7 +14,6 @@ from .sunrgbd_dataset import SUNRGBDDataset
 __all__ = [
     'KittiDataset', 'GroupSampler', 'DistributedGroupSampler',
     'build_dataloader', 'RepeatFactorDataset', 'DATASETS', 'build_dataset',
-    'build_dataloader'
     'CocoDataset', 'Kitti2DDataset', 'NuScenesDataset', 'ObjectSample',
     'RandomFlip3D', 'ObjectNoise', 'GlobalRotScaleTrans', 'PointShuffle',
     'ObjectRangeFilter', 'PointsRangeFilter', 'Collect3D',

mmdet3d/datasets/custom_3d.py

@@ -6,8 +6,7 @@ import numpy as np
 from torch.utils.data import Dataset
 
 from mmdet.datasets import DATASETS
-from ..core.bbox import (Box3DMode, CameraInstance3DBoxes,
-                         DepthInstance3DBoxes, LiDARInstance3DBoxes)
+from ..core.bbox import get_box_type
 from .pipelines import Compose
 
 
@@ -55,7 +54,7 @@ class Custom3DDataset(Dataset):
         self.test_mode = test_mode
         self.modality = modality
         self.filter_empty_gt = filter_empty_gt
-        self.get_box_type(box_type_3d)
+        self.box_type_3d, self.box_mode_3d = get_box_type(box_type_3d)
 
         self.CLASSES = self.get_classes(classes)
         self.data_infos = self.load_annotations(self.ann_file)
@@ -70,21 +69,6 @@ class Custom3DDataset(Dataset):
     def load_annotations(self, ann_file):
         return mmcv.load(ann_file)
 
-    def get_box_type(self, box_type):
-        box_type_lower = box_type.lower()
-        if box_type_lower == 'lidar':
-            self.box_type_3d = LiDARInstance3DBoxes
-            self.box_mode_3d = Box3DMode.LIDAR
-        elif box_type_lower == 'camera':
-            self.box_type_3d = CameraInstance3DBoxes
-            self.box_mode_3d = Box3DMode.CAM
-        elif box_type_lower == 'depth':
-            self.box_type_3d = DepthInstance3DBoxes
-            self.box_mode_3d = Box3DMode.DEPTH
-        else:
-            raise ValueError('Only "box_type" of "camera", "lidar", "depth"'
-                             f' are supported, got {box_type}')
-
     def get_data_info(self, index):
         info = self.data_infos[index]
         sample_idx = info['point_cloud']['lidar_idx']