# Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple, Union from mmcv.cnn import build_activation_layer, build_conv_layer, build_norm_layer from mmdet.models.backbones.resnet import BasicBlock, Bottleneck from torch import nn from mmdet3d.utils import ConfigType, OptConfigType from .spconv import IS_SPCONV2_AVAILABLE if IS_SPCONV2_AVAILABLE: from spconv.pytorch import SparseConvTensor, SparseModule, SparseSequential else: from mmcv.ops import (SparseConvTensor, SparseModule, SparseSequential, SparseConv3d, SparseInverseConv3d, SubMConv3d) from mmengine.model import BaseModule def replace_feature(out: SparseConvTensor, new_features: SparseConvTensor) -> SparseConvTensor: if 'replace_feature' in out.__dir__(): # spconv 2.x behaviour return out.replace_feature(new_features) else: out.features = new_features return out class SparseBottleneck(Bottleneck, SparseModule): """Sparse bottleneck block for PartA^2. Bottleneck block implemented with submanifold sparse convolution. Args: inplanes (int): Inplanes of block. planes (int): Planes of block. stride (int or Tuple[int]): Stride of the first block. Defaults to 1. downsample (Module, optional): Down sample module for block. Defaults to None. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for normalization layer. Defaults to None. """ expansion = 4 def __init__(self, inplanes: int, planes: int, stride: Union[int, Tuple[int]] = 1, downsample: nn.Module = None, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None) -> None: SparseModule.__init__(self) Bottleneck.__init__( self, inplanes, planes, stride=stride, downsample=downsample, conv_cfg=conv_cfg, norm_cfg=norm_cfg) def forward(self, x: SparseConvTensor) -> SparseConvTensor: identity = x.features out = self.conv1(x) out = replace_feature(out, self.bn1(out.features)) out = replace_feature(out, self.relu(out.features)) out = self.conv2(out) out = replace_feature(out, self.bn2(out.features)) out = replace_feature(out, self.relu(out.features)) out = self.conv3(out) out = replace_feature(out, self.bn3(out.features)) if self.downsample is not None: identity = self.downsample(x) out = replace_feature(out, out.features + identity) out = replace_feature(out, self.relu(out.features)) return out class SparseBasicBlock(BasicBlock, SparseModule): """Sparse basic block for PartA^2. Sparse basic block implemented with submanifold sparse convolution. Args: inplanes (int): Inplanes of block. planes (int): Planes of block. stride (int or Tuple[int]): Stride of the first block. Defaults to 1. downsample (Module, optional): Down sample module for block. Defaults to None. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for normalization layer. Defaults to None. """ expansion = 1 def __init__(self, inplanes: int, planes: int, stride: Union[int, Tuple[int]] = 1, downsample: nn.Module = None, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None) -> None: SparseModule.__init__(self) BasicBlock.__init__( self, inplanes, planes, stride=stride, downsample=downsample, conv_cfg=conv_cfg, norm_cfg=norm_cfg) def forward(self, x: SparseConvTensor) -> SparseConvTensor: identity = x.features assert x.features.dim() == 2, f'x.features.dim()={x.features.dim()}' out = self.conv1(x) out = replace_feature(out, self.norm1(out.features)) out = replace_feature(out, self.relu(out.features)) out = self.conv2(out) out = replace_feature(out, self.norm2(out.features)) if self.downsample is not None: identity = self.downsample(x) out = replace_feature(out, out.features + identity) out = replace_feature(out, self.relu(out.features)) return out def make_sparse_convmodule( in_channels: int, out_channels: int, kernel_size: Union[int, Tuple[int]], indice_key: str, stride: Union[int, Tuple[int]] = 1, padding: Union[int, Tuple[int]] = 0, conv_type: str = 'SubMConv3d', norm_cfg: OptConfigType = None, order: Tuple[str] = ('conv', 'norm', 'act') ) -> SparseSequential: """Make sparse convolution module. Args: in_channels (int): The number of input channels. out_channels (int): The number of out channels. kernel_size (int | Tuple[int]): Kernel size of convolution. indice_key (str): The indice key used for sparse tensor. stride (int or tuple[int]): The stride of convolution. padding (int or tuple[int]): The padding number of input. conv_type (str): Sparse conv type in spconv. Defaults to 'SubMConv3d'. norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for normalization layer. Defaults to None. order (Tuple[str]): The order of conv/norm/activation layers. It is a sequence of "conv", "norm" and "act". Common examples are ("conv", "norm", "act") and ("act", "conv", "norm"). Defaults to ('conv', 'norm', 'act'). Returns: spconv.SparseSequential: sparse convolution module. """ assert isinstance(order, tuple) and len(order) <= 3 assert set(order) | {'conv', 'norm', 'act'} == {'conv', 'norm', 'act'} conv_cfg = dict(type=conv_type, indice_key=indice_key) layers = list() for layer in order: if layer == 'conv': if conv_type not in [ 'SparseInverseConv3d', 'SparseInverseConv2d', 'SparseInverseConv1d' ]: layers.append( build_conv_layer( conv_cfg, in_channels, out_channels, kernel_size, stride=stride, padding=padding, bias=False)) else: layers.append( build_conv_layer( conv_cfg, in_channels, out_channels, kernel_size, bias=False)) elif layer == 'norm': layers.append(build_norm_layer(norm_cfg, out_channels)[1]) elif layer == 'act': layers.append(nn.ReLU(inplace=True)) layers = SparseSequential(*layers) return layers # The following module only supports spconv_v1 class AsymmResBlock(BaseModule): """Asymmetrical Residual Block. Args: in_channels (int): Input channels of the block. out_channels (int): Output channels of the block. norm_cfg (:obj:`ConfigDict` or dict): Config dict for normalization layer. act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers. Defaults to dict(type='LeakyReLU'). indice_key (str, optional): Name of indice tables. Defaults to None. """ def __init__(self, in_channels: int, out_channels: int, norm_cfg: ConfigType, act_cfg: ConfigType = dict(type='LeakyReLU'), indice_key: Optional[str] = None): super().__init__() self.conv0_0 = SubMConv3d( in_channels, out_channels, kernel_size=(1, 3, 3), padding=1, bias=False, indice_key=indice_key + 'bef') self.act0_0 = build_activation_layer(act_cfg) self.bn0_0 = build_norm_layer(norm_cfg, out_channels)[1] self.conv0_1 = SubMConv3d( out_channels, out_channels, kernel_size=(3, 1, 3), padding=1, bias=False, indice_key=indice_key + 'bef') self.act0_1 = build_activation_layer(act_cfg) self.bn0_1 = build_norm_layer(norm_cfg, out_channels)[1] self.conv1_0 = SubMConv3d( in_channels, out_channels, kernel_size=(3, 1, 3), padding=1, bias=False, indice_key=indice_key + 'bef') self.act1_0 = build_activation_layer(act_cfg) self.bn1_0 = build_norm_layer(norm_cfg, out_channels)[1] self.conv1_1 = SubMConv3d( out_channels, out_channels, kernel_size=(1, 3, 3), padding=1, bias=False, indice_key=indice_key + 'bef') self.act1_1 = build_activation_layer(act_cfg) self.bn1_1 = build_norm_layer(norm_cfg, out_channels)[1] def forward(self, x: SparseConvTensor) -> SparseConvTensor: """Forward pass.""" shortcut = self.conv0_0(x) shortcut.features = self.act0_0(shortcut.features) shortcut.features = self.bn0_0(shortcut.features) shortcut = self.conv0_1(shortcut) shortcut.features = self.act0_1(shortcut.features) shortcut.features = self.bn0_1(shortcut.features) res = self.conv1_0(x) res.features = self.act1_0(res.features) res.features = self.bn1_0(res.features) res = self.conv1_1(res) res.features = self.act1_1(res.features) res.features = self.bn1_1(res.features) res.features = res.features + shortcut.features return res class AsymmeDownBlock(BaseModule): """Asymmetrical DownSample Block. Args: in_channels (int): Input channels of the block. out_channels (int): Output channels of the block. norm_cfg (:obj:`ConfigDict` or dict): Config dict for normalization layer. act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers. Defaults to dict(type='LeakyReLU'). pooling (bool): Whether pooling features at the end of block. Defaults: True. height_pooling (bool): Whether pooling features at the height dimension. Defaults: False. indice_key (str, optional): Name of indice tables. Defaults to None. """ def __init__(self, in_channels: int, out_channels: int, norm_cfg: ConfigType, act_cfg: ConfigType = dict(type='LeakyReLU'), pooling: bool = True, height_pooling: bool = False, indice_key: Optional[str] = None): super().__init__() self.pooling = pooling self.conv0_0 = SubMConv3d( in_channels, out_channels, kernel_size=(3, 1, 3), padding=1, bias=False, indice_key=indice_key + 'bef') self.act0_0 = build_activation_layer(act_cfg) self.bn0_0 = build_norm_layer(norm_cfg, out_channels)[1] self.conv0_1 = SubMConv3d( out_channels, out_channels, kernel_size=(1, 3, 3), padding=1, bias=False, indice_key=indice_key + 'bef') self.act0_1 = build_activation_layer(act_cfg) self.bn0_1 = build_norm_layer(norm_cfg, out_channels)[1] self.conv1_0 = SubMConv3d( in_channels, out_channels, kernel_size=(1, 3, 3), padding=1, bias=False, indice_key=indice_key + 'bef') self.act1_0 = build_activation_layer(act_cfg) self.bn1_0 = build_norm_layer(norm_cfg, out_channels)[1] self.conv1_1 = SubMConv3d( out_channels, out_channels, kernel_size=(3, 1, 3), padding=1, bias=False, indice_key=indice_key + 'bef') self.act1_1 = build_activation_layer(act_cfg) self.bn1_1 = build_norm_layer(norm_cfg, out_channels)[1] if pooling: if height_pooling: self.pool = SparseConv3d( out_channels, out_channels, kernel_size=3, stride=2, padding=1, indice_key=indice_key, bias=False) else: self.pool = SparseConv3d( out_channels, out_channels, kernel_size=3, stride=(2, 2, 1), padding=1, indice_key=indice_key, bias=False) def forward(self, x: SparseConvTensor) -> SparseConvTensor: """Forward pass.""" shortcut = self.conv0_0(x) shortcut.features = self.act0_0(shortcut.features) shortcut.features = self.bn0_0(shortcut.features) shortcut = self.conv0_1(shortcut) shortcut.features = self.act0_1(shortcut.features) shortcut.features = self.bn0_1(shortcut.features) res = self.conv1_0(x) res.features = self.act1_0(res.features) res.features = self.bn1_0(res.features) res = self.conv1_1(res) res.features = self.act1_1(res.features) res.features = self.bn1_1(res.features) res.features = res.features + shortcut.features if self.pooling: pooled_res = self.pool(res) return pooled_res, res else: return res class AsymmeUpBlock(BaseModule): """Asymmetrical UpSample Block. Args: in_channels (int): Input channels of the block. out_channels (int): Output channels of the block. norm_cfg (:obj:`ConfigDict` or dict): Config dict for normalization layer. act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers. Defaults to dict(type='LeakyReLU'). indice_key (str, optional): Name of indice tables. Defaults to None. up_key (str, optional): Name of indice tables used in SparseInverseConv3d. Defaults to None. """ def __init__(self, in_channels: int, out_channels: int, norm_cfg: ConfigType, act_cfg: ConfigType = dict(type='LeakyReLU'), indice_key: Optional[str] = None, up_key: Optional[str] = None): super().__init__() self.trans_conv = SubMConv3d( in_channels, out_channels, kernel_size=(3, 3, 3), padding=1, bias=False, indice_key=indice_key + 'new_up') self.trans_act = build_activation_layer(act_cfg) self.trans_bn = build_norm_layer(norm_cfg, out_channels)[1] self.conv1 = SubMConv3d( out_channels, out_channels, kernel_size=(1, 3, 3), padding=1, bias=False, indice_key=indice_key) self.act1 = build_activation_layer(act_cfg) self.bn1 = build_norm_layer(norm_cfg, out_channels)[1] self.conv2 = SubMConv3d( out_channels, out_channels, kernel_size=(3, 1, 3), padding=1, bias=False, indice_key=indice_key) self.act2 = build_activation_layer(act_cfg) self.bn2 = build_norm_layer(norm_cfg, out_channels)[1] self.conv3 = SubMConv3d( out_channels, out_channels, kernel_size=(3, 3, 3), padding=1, bias=False, indice_key=indice_key) self.act3 = build_activation_layer(act_cfg) self.bn3 = build_norm_layer(norm_cfg, out_channels)[1] self.up_subm = SparseInverseConv3d( out_channels, out_channels, kernel_size=3, indice_key=up_key, bias=False) def forward(self, x: SparseConvTensor, skip: SparseConvTensor) -> SparseConvTensor: """Forward pass.""" x_trans = self.trans_conv(x) x_trans.features = self.trans_act(x_trans.features) x_trans.features = self.trans_bn(x_trans.features) # upsample up = self.up_subm(x_trans) up.features = up.features + skip.features up = self.conv1(up) up.features = self.act1(up.features) up.features = self.bn1(up.features) up = self.conv2(up) up.features = self.act2(up.features) up.features = self.bn2(up.features) up = self.conv3(up) up.features = self.act3(up.features) up.features = self.bn3(up.features) return up class DDCMBlock(BaseModule): """Dimension-Decomposition based Context Modeling. Args: in_channels (int): Input channels of the block. out_channels (int): Output channels of the block. norm_cfg (:obj:`ConfigDict` or dict): Config dict for normalization layer. act_cfg (:obj:`ConfigDict` or dict): Config dict of activation layers. Defaults to dict(type='Sigmoid'). indice_key (str, optional): Name of indice tables. Defaults to None. """ def __init__(self, in_channels: int, out_channels: int, norm_cfg: ConfigType, act_cfg: ConfigType = dict(type='Sigmoid'), indice_key: Optional[str] = None): super().__init__() self.conv1 = SubMConv3d( in_channels, out_channels, kernel_size=(3, 1, 1), padding=1, bias=False, indice_key=indice_key) self.bn1 = build_norm_layer(norm_cfg, out_channels)[1] self.act1 = build_activation_layer(act_cfg) self.conv2 = SubMConv3d( in_channels, out_channels, kernel_size=(1, 3, 1), padding=1, bias=False, indice_key=indice_key) self.bn2 = build_norm_layer(norm_cfg, out_channels)[1] self.act2 = build_activation_layer(act_cfg) self.conv3 = SubMConv3d( in_channels, out_channels, kernel_size=(1, 1, 3), padding=1, bias=False, indice_key=indice_key) self.bn3 = build_norm_layer(norm_cfg, out_channels)[1] self.act3 = build_activation_layer(act_cfg) def forward(self, x: SparseConvTensor) -> SparseConvTensor: """Forward pass.""" shortcut = self.conv1(x) shortcut.features = self.bn1(shortcut.features) shortcut.features = self.act1(shortcut.features) shortcut2 = self.conv2(x) shortcut2.features = self.bn2(shortcut2.features) shortcut2.features = self.act2(shortcut2.features) shortcut3 = self.conv3(x) shortcut3.features = self.bn3(shortcut3.features) shortcut3.features = self.act3(shortcut3.features) shortcut.features = shortcut.features + \ shortcut2.features + shortcut3.features shortcut.features = shortcut.features * x.features return shortcut