[Enhancement] Support SiLU with torch < 1.7.0 (#2278)

* support silu torch<1.7.0

* fix test

* fix test

* fix inplace

* fix inplace

* Update tests/test_cnn/test_silu.py
Co-authored-by: Zaida Zhou <58739961+zhouzaida@users.noreply.github.com>
Co-authored-by: Zaida Zhou <58739961+zhouzaida@users.noreply.github.com>

mmcv/cnn/bricks/activation.py

@@ -15,7 +15,23 @@ for module in [
     ACTIVATION_LAYERS.register_module(module=module)
 
 if digit_version(torch.__version__) >= digit_version('1.7.0'):
-    ACTIVATION_LAYERS.register_module(module=nn.SiLU)
+    ACTIVATION_LAYERS.register_module(module=nn.SiLU, name='SiLU')
+else:
+
+    class SiLU(nn.Module):
+        """Sigmoid Weighted Liner Unit."""
+
+        def __init__(self, inplace=False):
+            super().__init__()
+            self.inplace = inplace
+
+        def forward(self, inputs) -> torch.Tensor:
+            if self.inplace:
+                return inputs.mul_(torch.sigmoid(inputs))
+            else:
+                return inputs * torch.sigmoid(inputs)
+
+    ACTIVATION_LAYERS.register_module(module=SiLU, name='SiLU')
 
 
 @ACTIVATION_LAYERS.register_module(name='Clip')

tests/test_cnn/test_silu.py

 # Copyright (c) OpenMMLab. All rights reserved.
-import pytest
 import torch
-import torch.nn.functional as F
 
 from mmcv.cnn.bricks import build_activation_layer
-from mmcv.utils import digit_version
 
 
-@pytest.mark.skipif(
-    digit_version(torch.__version__) < digit_version('1.7.0'),
-    reason='torch.nn.SiLU is not available before 1.7.0')
 def test_silu():
     act = build_activation_layer(dict(type='SiLU'))
     input = torch.randn(1, 3, 64, 64)
-    expected_output = F.silu(input)
+    expected_output = input * torch.sigmoid(input)
     output = act(input)
     # test output shape
     assert output.shape == expected_output.shape
     # test output value
-    assert torch.equal(output, expected_output)
+    assert torch.allclose(output, expected_output)
+
+    # test inplace
+    act = build_activation_layer(dict(type='SiLU', inplace=True))
+    assert act.inplace
+    input = torch.randn(1, 3, 64, 64)
+    expected_output = input * torch.sigmoid(input)
+    output = act(input)
+    # test output shape
+    assert output.shape == expected_output.shape
+    # test output value
+    assert torch.allclose(output, expected_output)
+    assert torch.allclose(input, expected_output)
+    assert input is output