Unverified Commit 3e2de530 authored by xizaoqu's avatar xizaoqu Committed by GitHub
Browse files

[Fix] Fix cylinder3d (#2377)

* mv cylinder blocks from sparse blocks to backbone

* update
parent 4199bd61
......@@ -28,6 +28,8 @@ param_scheduler = [
gamma=0.1)
]
train_dataloader = dict(batch_size=4, )
# Default setting for scaling LR automatically
# - `enable` means enable scaling LR automatically
# or not by default.
......
......@@ -5,19 +5,389 @@ Please refer to `Cylinder3D github page
<https://github.com/xinge008/Cylinder3D>`_ for details
"""
from typing import List
from typing import List, Optional
import numpy as np
import torch
from mmcv.ops import SparseConvTensor
from mmcv.cnn import build_activation_layer, build_norm_layer
from mmcv.ops import (SparseConv3d, SparseConvTensor, SparseInverseConv3d,
SubMConv3d)
from mmengine.model import BaseModule
from mmdet3d.models.layers.sparse_block import (AsymmeDownBlock, AsymmeUpBlock,
AsymmResBlock, DDCMBlock)
from mmdet3d.registry import MODELS
from mmdet3d.utils import ConfigType
class AsymmResBlock(BaseModule):
"""Asymmetrical Residual Block.
Args:
in_channels (int): Input channels of the block.
out_channels (int): Output channels of the block.
norm_cfg (:obj:`ConfigDict` or dict): Config dict for
normalization layer.
act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers.
Defaults to dict(type='LeakyReLU').
indice_key (str, optional): Name of indice tables. Defaults to None.
"""
def __init__(self,
in_channels: int,
out_channels: int,
norm_cfg: ConfigType,
act_cfg: ConfigType = dict(type='LeakyReLU'),
indice_key: Optional[str] = None):
super().__init__()
self.conv0_0 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(1, 3, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act0_0 = build_activation_layer(act_cfg)
self.bn0_0 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv0_1 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(3, 1, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act0_1 = build_activation_layer(act_cfg)
self.bn0_1 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv1_0 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(3, 1, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act1_0 = build_activation_layer(act_cfg)
self.bn1_0 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv1_1 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(1, 3, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act1_1 = build_activation_layer(act_cfg)
self.bn1_1 = build_norm_layer(norm_cfg, out_channels)[1]
def forward(self, x: SparseConvTensor) -> SparseConvTensor:
"""Forward pass."""
shortcut = self.conv0_0(x)
shortcut.features = self.act0_0(shortcut.features)
shortcut.features = self.bn0_0(shortcut.features)
shortcut = self.conv0_1(shortcut)
shortcut.features = self.act0_1(shortcut.features)
shortcut.features = self.bn0_1(shortcut.features)
res = self.conv1_0(x)
res.features = self.act1_0(res.features)
res.features = self.bn1_0(res.features)
res = self.conv1_1(res)
res.features = self.act1_1(res.features)
res.features = self.bn1_1(res.features)
res.features = res.features + shortcut.features
return res
class AsymmeDownBlock(BaseModule):
"""Asymmetrical DownSample Block.
Args:
in_channels (int): Input channels of the block.
out_channels (int): Output channels of the block.
norm_cfg (:obj:`ConfigDict` or dict): Config dict for
normalization layer.
act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers.
Defaults to dict(type='LeakyReLU').
pooling (bool): Whether pooling features at the end of
block. Defaults: True.
height_pooling (bool): Whether pooling features at
the height dimension. Defaults: False.
indice_key (str, optional): Name of indice tables. Defaults to None.
"""
def __init__(self,
in_channels: int,
out_channels: int,
norm_cfg: ConfigType,
act_cfg: ConfigType = dict(type='LeakyReLU'),
pooling: bool = True,
height_pooling: bool = False,
indice_key: Optional[str] = None):
super().__init__()
self.pooling = pooling
self.conv0_0 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(3, 1, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act0_0 = build_activation_layer(act_cfg)
self.bn0_0 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv0_1 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(1, 3, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act0_1 = build_activation_layer(act_cfg)
self.bn0_1 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv1_0 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(1, 3, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act1_0 = build_activation_layer(act_cfg)
self.bn1_0 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv1_1 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(3, 1, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act1_1 = build_activation_layer(act_cfg)
self.bn1_1 = build_norm_layer(norm_cfg, out_channels)[1]
if pooling:
if height_pooling:
self.pool = SparseConv3d(
out_channels,
out_channels,
kernel_size=3,
stride=2,
padding=1,
indice_key=indice_key,
bias=False)
else:
self.pool = SparseConv3d(
out_channels,
out_channels,
kernel_size=3,
stride=(2, 2, 1),
padding=1,
indice_key=indice_key,
bias=False)
def forward(self, x: SparseConvTensor) -> SparseConvTensor:
"""Forward pass."""
shortcut = self.conv0_0(x)
shortcut.features = self.act0_0(shortcut.features)
shortcut.features = self.bn0_0(shortcut.features)
shortcut = self.conv0_1(shortcut)
shortcut.features = self.act0_1(shortcut.features)
shortcut.features = self.bn0_1(shortcut.features)
res = self.conv1_0(x)
res.features = self.act1_0(res.features)
res.features = self.bn1_0(res.features)
res = self.conv1_1(res)
res.features = self.act1_1(res.features)
res.features = self.bn1_1(res.features)
res.features = res.features + shortcut.features
if self.pooling:
pooled_res = self.pool(res)
return pooled_res, res
else:
return res
class AsymmeUpBlock(BaseModule):
"""Asymmetrical UpSample Block.
Args:
in_channels (int): Input channels of the block.
out_channels (int): Output channels of the block.
norm_cfg (:obj:`ConfigDict` or dict): Config dict for
normalization layer.
act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers.
Defaults to dict(type='LeakyReLU').
indice_key (str, optional): Name of indice tables. Defaults to None.
up_key (str, optional): Name of indice tables used in
SparseInverseConv3d. Defaults to None.
"""
def __init__(self,
in_channels: int,
out_channels: int,
norm_cfg: ConfigType,
act_cfg: ConfigType = dict(type='LeakyReLU'),
indice_key: Optional[str] = None,
up_key: Optional[str] = None):
super().__init__()
self.trans_conv = SubMConv3d(
in_channels,
out_channels,
kernel_size=(3, 3, 3),
padding=1,
bias=False,
indice_key=indice_key + 'new_up')
self.trans_act = build_activation_layer(act_cfg)
self.trans_bn = build_norm_layer(norm_cfg, out_channels)[1]
self.conv1 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(1, 3, 3),
padding=1,
bias=False,
indice_key=indice_key)
self.act1 = build_activation_layer(act_cfg)
self.bn1 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv2 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(3, 1, 3),
padding=1,
bias=False,
indice_key=indice_key)
self.act2 = build_activation_layer(act_cfg)
self.bn2 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv3 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(3, 3, 3),
padding=1,
bias=False,
indice_key=indice_key)
self.act3 = build_activation_layer(act_cfg)
self.bn3 = build_norm_layer(norm_cfg, out_channels)[1]
self.up_subm = SparseInverseConv3d(
out_channels,
out_channels,
kernel_size=3,
indice_key=up_key,
bias=False)
def forward(self, x: SparseConvTensor,
skip: SparseConvTensor) -> SparseConvTensor:
"""Forward pass."""
x_trans = self.trans_conv(x)
x_trans.features = self.trans_act(x_trans.features)
x_trans.features = self.trans_bn(x_trans.features)
# upsample
up = self.up_subm(x_trans)
up.features = up.features + skip.features
up = self.conv1(up)
up.features = self.act1(up.features)
up.features = self.bn1(up.features)
up = self.conv2(up)
up.features = self.act2(up.features)
up.features = self.bn2(up.features)
up = self.conv3(up)
up.features = self.act3(up.features)
up.features = self.bn3(up.features)
return up
class DDCMBlock(BaseModule):
"""Dimension-Decomposition based Context Modeling.
Args:
in_channels (int): Input channels of the block.
out_channels (int): Output channels of the block.
norm_cfg (:obj:`ConfigDict` or dict): Config dict for
normalization layer.
act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers.
Defaults to dict(type='Sigmoid').
indice_key (str, optional): Name of indice tables. Defaults to None.
"""
def __init__(self,
in_channels: int,
out_channels: int,
norm_cfg: ConfigType,
act_cfg: ConfigType = dict(type='Sigmoid'),
indice_key: Optional[str] = None):
super().__init__()
self.conv1 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(3, 1, 1),
padding=1,
bias=False,
indice_key=indice_key)
self.bn1 = build_norm_layer(norm_cfg, out_channels)[1]
self.act1 = build_activation_layer(act_cfg)
self.conv2 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(1, 3, 1),
padding=1,
bias=False,
indice_key=indice_key)
self.bn2 = build_norm_layer(norm_cfg, out_channels)[1]
self.act2 = build_activation_layer(act_cfg)
self.conv3 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(1, 1, 3),
padding=1,
bias=False,
indice_key=indice_key)
self.bn3 = build_norm_layer(norm_cfg, out_channels)[1]
self.act3 = build_activation_layer(act_cfg)
def forward(self, x: SparseConvTensor) -> SparseConvTensor:
"""Forward pass."""
shortcut = self.conv1(x)
shortcut.features = self.bn1(shortcut.features)
shortcut.features = self.act1(shortcut.features)
shortcut2 = self.conv2(x)
shortcut2.features = self.bn2(shortcut2.features)
shortcut2.features = self.act2(shortcut2.features)
shortcut3 = self.conv3(x)
shortcut3.features = self.bn3(shortcut3.features)
shortcut3.features = self.act3(shortcut3.features)
shortcut.features = shortcut.features + \
shortcut2.features + shortcut3.features
shortcut.features = shortcut.features * x.features
return shortcut
@MODELS.register_module()
class Asymm3DSpconv(BaseModule):
"""Asymmetrical 3D convolution networks.
......
# Copyright (c) OpenMMLab. All rights reserved.
from typing import Optional, Tuple, Union
from typing import Tuple, Union
from mmcv.cnn import build_activation_layer, build_conv_layer, build_norm_layer
from mmcv.cnn import build_conv_layer, build_norm_layer
from mmdet.models.backbones.resnet import BasicBlock, Bottleneck
from torch import nn
from mmdet3d.utils import ConfigType, OptConfigType
from mmdet3d.utils import OptConfigType
from .spconv import IS_SPCONV2_AVAILABLE
if IS_SPCONV2_AVAILABLE:
from spconv.pytorch import SparseConvTensor, SparseModule, SparseSequential
else:
from mmcv.ops import (SparseConvTensor, SparseModule, SparseSequential,
SparseConv3d, SparseInverseConv3d, SubMConv3d)
from mmengine.model import BaseModule
from mmcv.ops import SparseConvTensor, SparseModule, SparseSequential
def replace_feature(out: SparseConvTensor,
......@@ -210,374 +207,3 @@ def make_sparse_convmodule(
layers = SparseSequential(*layers)
return layers
# The following module only supports spconv_v1
class AsymmResBlock(BaseModule):
"""Asymmetrical Residual Block.
Args:
in_channels (int): Input channels of the block.
out_channels (int): Output channels of the block.
norm_cfg (:obj:`ConfigDict` or dict): Config dict for
normalization layer.
act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers.
Defaults to dict(type='LeakyReLU').
indice_key (str, optional): Name of indice tables. Defaults to None.
"""
def __init__(self,
in_channels: int,
out_channels: int,
norm_cfg: ConfigType,
act_cfg: ConfigType = dict(type='LeakyReLU'),
indice_key: Optional[str] = None):
super().__init__()
self.conv0_0 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(1, 3, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act0_0 = build_activation_layer(act_cfg)
self.bn0_0 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv0_1 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(3, 1, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act0_1 = build_activation_layer(act_cfg)
self.bn0_1 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv1_0 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(3, 1, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act1_0 = build_activation_layer(act_cfg)
self.bn1_0 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv1_1 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(1, 3, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act1_1 = build_activation_layer(act_cfg)
self.bn1_1 = build_norm_layer(norm_cfg, out_channels)[1]
def forward(self, x: SparseConvTensor) -> SparseConvTensor:
"""Forward pass."""
shortcut = self.conv0_0(x)
shortcut.features = self.act0_0(shortcut.features)
shortcut.features = self.bn0_0(shortcut.features)
shortcut = self.conv0_1(shortcut)
shortcut.features = self.act0_1(shortcut.features)
shortcut.features = self.bn0_1(shortcut.features)
res = self.conv1_0(x)
res.features = self.act1_0(res.features)
res.features = self.bn1_0(res.features)
res = self.conv1_1(res)
res.features = self.act1_1(res.features)
res.features = self.bn1_1(res.features)
res.features = res.features + shortcut.features
return res
class AsymmeDownBlock(BaseModule):
"""Asymmetrical DownSample Block.
Args:
in_channels (int): Input channels of the block.
out_channels (int): Output channels of the block.
norm_cfg (:obj:`ConfigDict` or dict): Config dict for
normalization layer.
act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers.
Defaults to dict(type='LeakyReLU').
pooling (bool): Whether pooling features at the end of
block. Defaults: True.
height_pooling (bool): Whether pooling features at
the height dimension. Defaults: False.
indice_key (str, optional): Name of indice tables. Defaults to None.
"""
def __init__(self,
in_channels: int,
out_channels: int,
norm_cfg: ConfigType,
act_cfg: ConfigType = dict(type='LeakyReLU'),
pooling: bool = True,
height_pooling: bool = False,
indice_key: Optional[str] = None):
super().__init__()
self.pooling = pooling
self.conv0_0 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(3, 1, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act0_0 = build_activation_layer(act_cfg)
self.bn0_0 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv0_1 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(1, 3, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act0_1 = build_activation_layer(act_cfg)
self.bn0_1 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv1_0 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(1, 3, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act1_0 = build_activation_layer(act_cfg)
self.bn1_0 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv1_1 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(3, 1, 3),
padding=1,
bias=False,
indice_key=indice_key + 'bef')
self.act1_1 = build_activation_layer(act_cfg)
self.bn1_1 = build_norm_layer(norm_cfg, out_channels)[1]
if pooling:
if height_pooling:
self.pool = SparseConv3d(
out_channels,
out_channels,
kernel_size=3,
stride=2,
padding=1,
indice_key=indice_key,
bias=False)
else:
self.pool = SparseConv3d(
out_channels,
out_channels,
kernel_size=3,
stride=(2, 2, 1),
padding=1,
indice_key=indice_key,
bias=False)
def forward(self, x: SparseConvTensor) -> SparseConvTensor:
"""Forward pass."""
shortcut = self.conv0_0(x)
shortcut.features = self.act0_0(shortcut.features)
shortcut.features = self.bn0_0(shortcut.features)
shortcut = self.conv0_1(shortcut)
shortcut.features = self.act0_1(shortcut.features)
shortcut.features = self.bn0_1(shortcut.features)
res = self.conv1_0(x)
res.features = self.act1_0(res.features)
res.features = self.bn1_0(res.features)
res = self.conv1_1(res)
res.features = self.act1_1(res.features)
res.features = self.bn1_1(res.features)
res.features = res.features + shortcut.features
if self.pooling:
pooled_res = self.pool(res)
return pooled_res, res
else:
return res
class AsymmeUpBlock(BaseModule):
"""Asymmetrical UpSample Block.
Args:
in_channels (int): Input channels of the block.
out_channels (int): Output channels of the block.
norm_cfg (:obj:`ConfigDict` or dict): Config dict for
normalization layer.
act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers.
Defaults to dict(type='LeakyReLU').
indice_key (str, optional): Name of indice tables. Defaults to None.
up_key (str, optional): Name of indice tables used in
SparseInverseConv3d. Defaults to None.
"""
def __init__(self,
in_channels: int,
out_channels: int,
norm_cfg: ConfigType,
act_cfg: ConfigType = dict(type='LeakyReLU'),
indice_key: Optional[str] = None,
up_key: Optional[str] = None):
super().__init__()
self.trans_conv = SubMConv3d(
in_channels,
out_channels,
kernel_size=(3, 3, 3),
padding=1,
bias=False,
indice_key=indice_key + 'new_up')
self.trans_act = build_activation_layer(act_cfg)
self.trans_bn = build_norm_layer(norm_cfg, out_channels)[1]
self.conv1 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(1, 3, 3),
padding=1,
bias=False,
indice_key=indice_key)
self.act1 = build_activation_layer(act_cfg)
self.bn1 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv2 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(3, 1, 3),
padding=1,
bias=False,
indice_key=indice_key)
self.act2 = build_activation_layer(act_cfg)
self.bn2 = build_norm_layer(norm_cfg, out_channels)[1]
self.conv3 = SubMConv3d(
out_channels,
out_channels,
kernel_size=(3, 3, 3),
padding=1,
bias=False,
indice_key=indice_key)
self.act3 = build_activation_layer(act_cfg)
self.bn3 = build_norm_layer(norm_cfg, out_channels)[1]
self.up_subm = SparseInverseConv3d(
out_channels,
out_channels,
kernel_size=3,
indice_key=up_key,
bias=False)
def forward(self, x: SparseConvTensor,
skip: SparseConvTensor) -> SparseConvTensor:
"""Forward pass."""
x_trans = self.trans_conv(x)
x_trans.features = self.trans_act(x_trans.features)
x_trans.features = self.trans_bn(x_trans.features)
# upsample
up = self.up_subm(x_trans)
up.features = up.features + skip.features
up = self.conv1(up)
up.features = self.act1(up.features)
up.features = self.bn1(up.features)
up = self.conv2(up)
up.features = self.act2(up.features)
up.features = self.bn2(up.features)
up = self.conv3(up)
up.features = self.act3(up.features)
up.features = self.bn3(up.features)
return up
class DDCMBlock(BaseModule):
"""Dimension-Decomposition based Context Modeling.
Args:
in_channels (int): Input channels of the block.
out_channels (int): Output channels of the block.
norm_cfg (:obj:`ConfigDict` or dict): Config dict for
normalization layer.
act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers.
Defaults to dict(type='Sigmoid').
indice_key (str, optional): Name of indice tables. Defaults to None.
"""
def __init__(self,
in_channels: int,
out_channels: int,
norm_cfg: ConfigType,
act_cfg: ConfigType = dict(type='Sigmoid'),
indice_key: Optional[str] = None):
super().__init__()
self.conv1 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(3, 1, 1),
padding=1,
bias=False,
indice_key=indice_key)
self.bn1 = build_norm_layer(norm_cfg, out_channels)[1]
self.act1 = build_activation_layer(act_cfg)
self.conv2 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(1, 3, 1),
padding=1,
bias=False,
indice_key=indice_key)
self.bn2 = build_norm_layer(norm_cfg, out_channels)[1]
self.act2 = build_activation_layer(act_cfg)
self.conv3 = SubMConv3d(
in_channels,
out_channels,
kernel_size=(1, 1, 3),
padding=1,
bias=False,
indice_key=indice_key)
self.bn3 = build_norm_layer(norm_cfg, out_channels)[1]
self.act3 = build_activation_layer(act_cfg)
def forward(self, x: SparseConvTensor) -> SparseConvTensor:
"""Forward pass."""
shortcut = self.conv1(x)
shortcut.features = self.bn1(shortcut.features)
shortcut.features = self.act1(shortcut.features)
shortcut2 = self.conv2(x)
shortcut2.features = self.bn2(shortcut2.features)
shortcut2.features = self.act2(shortcut2.features)
shortcut3 = self.conv3(x)
shortcut3.features = self.bn3(shortcut3.features)
shortcut3.features = self.act3(shortcut3.features)
shortcut.features = shortcut.features + \
shortcut2.features + shortcut3.features
shortcut.features = shortcut.features * x.features
return shortcut
......@@ -17,7 +17,7 @@ class TestCylinder3D(unittest.TestCase):
DefaultScope.get_instance('test_cylinder3d', scope_name='mmdet3d')
setup_seed(0)
cylinder3d_cfg = get_detector_cfg(
'cylinder3d/cylinder3d_4xb2_3x_semantickitti.py')
'cylinder3d/cylinder3d_4xb4_3x_semantickitti.py')
cylinder3d_cfg.decode_head['ignore_index'] = 1
model = MODELS.build(cylinder3d_cfg)
num_gt_instance = 3
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment