Merge branch 'refactor-secfpn' into 'master'

Refactor SECOND FPN

See merge request open-mmlab/mmdet.3d!25

.gitlab-ci.yml

@@ -26,6 +26,7 @@ before_script:
   script:
     - echo "Start building..."
     - pip install "git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI"
+    - pip install git+https://github.com/open-mmlab/mmcv.git
     - pip install git+https://github.com/open-mmlab/mmdetection.git
     - python -c "import mmdet; print(mmdet.__version__)"
     - pip install -v -e .[all]

configs/kitti/dv_mvx-v2_second_secfpn_fpn-fusion_adamw_2x8_80e_kitti-3d-3class.py

@@ -63,7 +63,7 @@ model = dict(
         type='SECONDFPN',
         in_channels=[128, 256],
         upsample_strides=[1, 2],
-        num_upsample_filters=[256, 256],
+        out_channels=[256, 256],
     ),
     pts_bbox_head=dict(
         type='SECONDHead',

configs/kitti/dv_pointpillars_secfpn_6x8_160e_kitti-3d-car.py

@@ -34,7 +34,7 @@ model = dict(
         type='SECONDFPN',
         in_channels=[64, 128, 256],
         upsample_strides=[1, 2, 4],
-        num_upsample_filters=[128, 128, 128],
+        out_channels=[128, 128, 128],
     ),
     bbox_head=dict(
         type='SECONDHead',

configs/kitti/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py

@@ -32,7 +32,7 @@ model = dict(
         type='SECONDFPN',
         in_channels=[128, 256],
         upsample_strides=[1, 2],
-        num_upsample_filters=[256, 256],
+        out_channels=[256, 256],
     ),
     bbox_head=dict(
         type='SECONDHead',

configs/kitti/dv_second_secfpn_6x8_80e_kitti-3d-car.py

@@ -32,7 +32,7 @@ model = dict(
         type='SECONDFPN',
         in_channels=[128, 256],
         upsample_strides=[1, 2],
-        num_upsample_filters=[256, 256],
+        out_channels=[256, 256],
     ),
     bbox_head=dict(
         type='SECONDHead',

configs/kitti/hv_PartA2_secfpn_4x8_cosine_80e_kitti-3d-3class.py

@@ -27,7 +27,7 @@ model = dict(
         type='SECONDFPN',
         in_channels=[128, 256],
         upsample_strides=[1, 2],
-        num_upsample_filters=[256, 256]),
+        out_channels=[256, 256]),
     rpn_head=dict(
         type='PartA2RPNHead',
         class_name=['Pedestrian', 'Cyclist', 'Car'],

configs/kitti/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car.py

@@ -33,7 +33,7 @@ model = dict(
         type='SECONDFPN',
         in_channels=[64, 128, 256],
         upsample_strides=[1, 2, 4],
-        num_upsample_filters=[128, 128, 128],
+        out_channels=[128, 128, 128],
     ),
     bbox_head=dict(
         type='SECONDHead',

configs/kitti/hv_second_secfpn_6x8_80e_kitti-3d-car.py

@@ -32,7 +32,7 @@ model = dict(
         type='SECONDFPN',
         in_channels=[128, 256],
         upsample_strides=[1, 2],
-        num_upsample_filters=[256, 256],
+        out_channels=[256, 256],
     ),
     bbox_head=dict(
         type='SECONDHead',

configs/nus/hv_pointpillars_secfpn_sbn-all_4x8_2x_nus-3d.py

@@ -41,7 +41,7 @@ model = dict(
         norm_cfg=dict(type='naiveSyncBN2d', eps=1e-3, momentum=0.01),
         in_channels=[64, 128, 256],
         upsample_strides=[1, 2, 4],
-        num_upsample_filters=[128, 128, 128],
+        out_channels=[128, 128, 128],
     ),
     pts_bbox_head=dict(
         type='Anchor3DVeloHead',

mmdet3d/models/necks/second_fpn.py

-from functools import partial
-
 import torch
 import torch.nn as nn
-from mmcv.cnn import build_norm_layer, constant_init, kaiming_init
-from torch.nn import Sequential
-from torch.nn.modules.batchnorm import _BatchNorm
+from mmcv.cnn import (build_norm_layer, build_upsample_layer, constant_init,
+                      is_norm, kaiming_init)
 
 from mmdet.models import NECKS
 from .. import builder
@@ -12,36 +9,40 @@ from .. import builder
 
 @NECKS.register_module()
 class SECONDFPN(nn.Module):
-    """Compare with RPN, RPNV2 support arbitrary number of stage.
+    """FPN used in SECOND/PointPillars/PartA2/MVXNet
+
+    Args:
+        in_channels (list[int]): Input channels of multi-scale feature maps
+        out_channels (list[int]): Output channels of feature maps
+        upsample_strides (list[int]): Strides used to upsample the feature maps
+        norm_cfg (dict): Config dict of normalization layers
+        upsample_cfg (dict): Config dict of upsample layers
     """
 
     def __init__(self,
-                 use_norm=True,
                  in_channels=[128, 128, 256],
+                 out_channels=[256, 256, 256],
                  upsample_strides=[1, 2, 4],
-                 num_upsample_filters=[256, 256, 256],
-                 norm_cfg=dict(type='BN', eps=1e-3, momentum=0.01)):
+                 norm_cfg=dict(type='BN', eps=1e-3, momentum=0.01),
+                 upsample_cfg=dict(type='deconv', bias=False)):
         # if for GroupNorm,
         # cfg is dict(type='GN', num_groups=num_groups, eps=1e-3, affine=True)
         super(SECONDFPN, self).__init__()
-        assert len(num_upsample_filters) == len(upsample_strides)
+        assert len(out_channels) == len(upsample_strides) == len(in_channels)
         self.in_channels = in_channels
-
-        ConvTranspose2d = partial(nn.ConvTranspose2d, bias=False)
+        self.out_channels = out_channels
 
         deblocks = []
-
-        for i, num_upsample_filter in enumerate(num_upsample_filters):
-            norm_layer = build_norm_layer(norm_cfg, num_upsample_filter)[1]
-            deblock = Sequential(
-                ConvTranspose2d(
-                    in_channels[i],
-                    num_upsample_filter,
-                    upsample_strides[i],
-                    stride=upsample_strides[i]),
-                norm_layer,
-                nn.ReLU(inplace=True),
-            )
+        for i, out_channel in enumerate(out_channels):
+            upsample_layer = build_upsample_layer(
+                upsample_cfg,
+                in_channels=in_channels[i],
+                out_channels=out_channel,
+                kernel_size=upsample_strides[i],
+                stride=upsample_strides[i])
+            deblock = nn.Sequential(upsample_layer,
+                                    build_norm_layer(norm_cfg, out_channel)[1],
+                                    nn.ReLU(inplace=True))
             deblocks.append(deblock)
         self.deblocks = nn.ModuleList(deblocks)
 
@@ -49,7 +50,7 @@ class SECONDFPN(nn.Module):
         for m in self.modules():
             if isinstance(m, nn.Conv2d):
                 kaiming_init(m)
-            elif isinstance(m, (_BatchNorm, nn.GroupNorm)):
+            elif is_norm(m):
                 constant_init(m, 1)
 
     def forward(self, x):
@@ -65,30 +66,34 @@ class SECONDFPN(nn.Module):
 
 @NECKS.register_module()
 class SECONDFusionFPN(SECONDFPN):
-    """Compare with RPN, RPNV2 support arbitrary number of stage.
+    """FPN used in multi-modality SECOND/PointPillars
+
+    Args:
+        in_channels (list[int]): Input channels of multi-scale feature maps
+        out_channels (list[int]): Output channels of feature maps
+        upsample_strides (list[int]): Strides used to upsample the feature maps
+        norm_cfg (dict): Config dict of normalization layers
+        upsample_cfg (dict): Config dict of upsample layers
+        downsample_rates (list[int]): The downsample rate of feature map in
+            comparison to the original voxelization input
+        fusion_layer (dict): Config dict of fusion layers
     """
 
     def __init__(self,
-                 use_norm=True,
                  in_channels=[128, 128, 256],
+                 out_channels=[256, 256, 256],
                  upsample_strides=[1, 2, 4],
-                 num_upsample_filters=[256, 256, 256],
                  norm_cfg=dict(type='BN', eps=1e-3, momentum=0.01),
-                 down_sample_rate=[40, 8, 8],
-                 fusion_layer=None,
-                 cat_points=False):
-        super(SECONDFusionFPN, self).__init__(
-            use_norm,
-            in_channels,
-            upsample_strides,
-            num_upsample_filters,
-            norm_cfg,
-        )
+                 upsample_cfg=dict(type='deconv', bias=False),
+                 downsample_rates=[40, 8, 8],
+                 fusion_layer=None):
+        super(SECONDFusionFPN,
+              self).__init__(in_channels, out_channels, upsample_strides,
+                             norm_cfg, upsample_cfg)
         self.fusion_layer = None
         if fusion_layer is not None:
             self.fusion_layer = builder.build_fusion_layer(fusion_layer)
-        self.cat_points = cat_points
-        self.down_sample_rate = down_sample_rate
+        self.downsample_rates = downsample_rates
 
     def forward(self,
                 x,
@@ -107,11 +112,11 @@ class SECONDFusionFPN(SECONDFPN):
             downsample_pts_coors = torch.zeros_like(coors)
             downsample_pts_coors[:, 0] = coors[:, 0]
             downsample_pts_coors[:, 1] = (
-                coors[:, 1] / self.down_sample_rate[0])
+                coors[:, 1] / self.downsample_rates[0])
             downsample_pts_coors[:, 2] = (
-                coors[:, 2] / self.down_sample_rate[1])
+                coors[:, 2] / self.downsample_rates[1])
             downsample_pts_coors[:, 3] = (
-                coors[:, 3] / self.down_sample_rate[2])
+                coors[:, 3] / self.downsample_rates[2])
             # fusion for each point
             out = self.fusion_layer(img_feats, points, out,
                                     downsample_pts_coors, img_meta)

requirements/build.txt

 # These must be installed before building mmdetection
 numpy
-torch>=1.1
+torch>=1.3

requirements/runtime.txt

 matplotlib
-mmcv>=0.5.0
+mmcv>=0.5.1
 numba==0.45.1
 numpy
 # need older pillow until torchvision is fixed

tests/test_fpn.py

+import pytest
+
+
+def test_secfpn():
+    neck_cfg = dict(
+        type='SECONDFPN',
+        in_channels=[2, 3],
+        upsample_strides=[1, 2],
+        out_channels=[4, 6],
+    )
+    from mmdet.models.builder import build_neck
+    neck = build_neck(neck_cfg)
+    assert neck.deblocks[0][0].in_channels == 2
+    assert neck.deblocks[1][0].in_channels == 3
+    assert neck.deblocks[0][0].out_channels == 4
+    assert neck.deblocks[1][0].out_channels == 6
+    assert neck.deblocks[0][0].stride == (1, 1)
+    assert neck.deblocks[1][0].stride == (2, 2)
+    assert neck is not None
+
+    neck_cfg = dict(
+        type='SECONDFPN',
+        in_channels=[2, 2],
+        upsample_strides=[1, 2, 4],
+        out_channels=[2, 2],
+    )
+    with pytest.raises(AssertionError):
+        build_neck(neck_cfg)
+
+    neck_cfg = dict(
+        type='SECONDFPN',
+        in_channels=[2, 2, 4],
+        upsample_strides=[1, 2, 4],
+        out_channels=[2, 2],
+    )
+    with pytest.raises(AssertionError):
+        build_neck(neck_cfg)