resnet.py 15.9 KB
Newer Older
1
import logging
2
import pickle
3

4
import torch
Kai Chen's avatar
Kai Chen committed
5
6
import torch.nn as nn
import torch.utils.checkpoint as cp
Kai Chen's avatar
Kai Chen committed
7
8

from mmcv.cnn import constant_init, kaiming_init
Kai Chen's avatar
Kai Chen committed
9
from mmcv.runner import load_checkpoint
10
from ..utils import build_norm_layer
Kai Chen's avatar
Kai Chen committed
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33


def conv3x3(in_planes, out_planes, stride=1, dilation=1):
    "3x3 convolution with padding"
    return nn.Conv2d(
        in_planes,
        out_planes,
        kernel_size=3,
        stride=stride,
        padding=dilation,
        dilation=dilation,
        bias=False)


class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self,
                 inplanes,
                 planes,
                 stride=1,
                 dilation=1,
                 downsample=None,
Kai Chen's avatar
Kai Chen committed
34
                 style='pytorch',
35
                 with_cp=False,
ThangVu's avatar
ThangVu committed
36
                 normalize=dict(type='BN')):
Kai Chen's avatar
Kai Chen committed
37
38
        super(BasicBlock, self).__init__()
        self.conv1 = conv3x3(inplanes, planes, stride, dilation)
39
40
41
42
43
44
45
46
47

        norm_layers = []
        norm_layers.append(build_norm_layer(normalize, planes))
        norm_layers.append(build_norm_layer(normalize, planes))
        self.norm_names = (['gn1', 'gn2'] if normalize['type'] == 'GN'
                           else ['bn1', 'bn2'])
        for name, layer in zip(self.norm_names, norm_layers):
            self.add_module(name, layer)

Kai Chen's avatar
Kai Chen committed
48
49
50
51
52
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x3(planes, planes)
        self.downsample = downsample
        self.stride = stride
        self.dilation = dilation
Kai Chen's avatar
Kai Chen committed
53
        assert not with_cp
Kai Chen's avatar
Kai Chen committed
54
55

    def forward(self, x):
pangjm's avatar
pangjm committed
56
        identity = x
Kai Chen's avatar
Kai Chen committed
57
58

        out = self.conv1(x)
59
        out = getattr(self, self.norm_names[0])(out)
Kai Chen's avatar
Kai Chen committed
60
61
62
        out = self.relu(out)

        out = self.conv2(out)
63
        out = getattr(self, self.norm_names[1])(out)
Kai Chen's avatar
Kai Chen committed
64
65

        if self.downsample is not None:
pangjm's avatar
pangjm committed
66
            identity = self.downsample(x)
Kai Chen's avatar
Kai Chen committed
67

pangjm's avatar
pangjm committed
68
        out += identity
Kai Chen's avatar
Kai Chen committed
69
70
71
72
73
74
75
76
77
78
79
80
81
82
        out = self.relu(out)

        return out


class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self,
                 inplanes,
                 planes,
                 stride=1,
                 dilation=1,
                 downsample=None,
83
                 style='pytorch',
84
85
                 with_cp=False,
                 normalize=dict(type='BN')):
pangjm's avatar
pangjm committed
86
        """Bottleneck block for ResNet.
87
88
        If style is "pytorch", the stride-two layer is the 3x3 conv layer,
        if it is "caffe", the stride-two layer is the first 1x1 conv layer.
Kai Chen's avatar
Kai Chen committed
89
90
        """
        super(Bottleneck, self).__init__()
91
        assert style in ['pytorch', 'caffe']
pangjm's avatar
pangjm committed
92
93
        self.inplanes = inplanes
        self.planes = planes
94
        if style == 'pytorch':
pangjm's avatar
pangjm committed
95
96
            self.conv1_stride = 1
            self.conv2_stride = stride
Kai Chen's avatar
Kai Chen committed
97
        else:
pangjm's avatar
pangjm committed
98
99
            self.conv1_stride = stride
            self.conv2_stride = 1
Kai Chen's avatar
Kai Chen committed
100
        self.conv1 = nn.Conv2d(
pangjm's avatar
pangjm committed
101
102
103
104
105
            inplanes,
            planes,
            kernel_size=1,
            stride=self.conv1_stride,
            bias=False)
Kai Chen's avatar
Kai Chen committed
106
107
108
109
        self.conv2 = nn.Conv2d(
            planes,
            planes,
            kernel_size=3,
pangjm's avatar
pangjm committed
110
            stride=self.conv2_stride,
Kai Chen's avatar
Kai Chen committed
111
112
113
114
            padding=dilation,
            dilation=dilation,
            bias=False)

115
116
117
118
119
120
121
122
123
        norm_layers = []
        norm_layers.append(build_norm_layer(normalize, planes))
        norm_layers.append(build_norm_layer(normalize, planes))
        norm_layers.append(build_norm_layer(normalize, planes*self.expansion))
        self.norm_names = (['gn1', 'gn2', 'gn3'] if normalize['type'] == 'GN'
                           else ['bn1', 'bn2', 'bn3'])
        for name, layer in zip(self.norm_names, norm_layers):
            self.add_module(name, layer)

Kai Chen's avatar
Kai Chen committed
124
125
126
127
128
129
130
        self.conv3 = nn.Conv2d(
            planes, planes * self.expansion, kernel_size=1, bias=False)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride
        self.dilation = dilation
        self.with_cp = with_cp
131
        self.normalize = normalize
Kai Chen's avatar
Kai Chen committed
132
133
134
135

    def forward(self, x):

        def _inner_forward(x):
pangjm's avatar
pangjm committed
136
            identity = x
Kai Chen's avatar
Kai Chen committed
137
138

            out = self.conv1(x)
139
            out = getattr(self, self.norm_names[0])(out)
Kai Chen's avatar
Kai Chen committed
140
141
142
            out = self.relu(out)

            out = self.conv2(out)
143
            out = getattr(self, self.norm_names[1])(out)
Kai Chen's avatar
Kai Chen committed
144
145
146
            out = self.relu(out)

            out = self.conv3(out)
147
            out = getattr(self, self.norm_names[2])(out)
Kai Chen's avatar
Kai Chen committed
148
149

            if self.downsample is not None:
pangjm's avatar
pangjm committed
150
                identity = self.downsample(x)
Kai Chen's avatar
Kai Chen committed
151

pangjm's avatar
pangjm committed
152
            out += identity
Kai Chen's avatar
Kai Chen committed
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171

            return out

        if self.with_cp and x.requires_grad:
            out = cp.checkpoint(_inner_forward, x)
        else:
            out = _inner_forward(x)

        out = self.relu(out)

        return out


def make_res_layer(block,
                   inplanes,
                   planes,
                   blocks,
                   stride=1,
                   dilation=1,
172
                   style='pytorch',
173
174
                   with_cp=False,
                   normalize=dict(type='BN')):
Kai Chen's avatar
Kai Chen committed
175
176
177
178
179
180
181
182
183
    downsample = None
    if stride != 1 or inplanes != planes * block.expansion:
        downsample = nn.Sequential(
            nn.Conv2d(
                inplanes,
                planes * block.expansion,
                kernel_size=1,
                stride=stride,
                bias=False),
184
            build_norm_layer(normalize, planes * block.expansion),
Kai Chen's avatar
Kai Chen committed
185
186
187
188
189
190
191
192
193
194
195
        )

    layers = []
    layers.append(
        block(
            inplanes,
            planes,
            stride,
            dilation,
            downsample,
            style=style,
196
197
            with_cp=with_cp,
            normalize=normalize))
Kai Chen's avatar
Kai Chen committed
198
199
200
    inplanes = planes * block.expansion
    for i in range(1, blocks):
        layers.append(
201
202
            block(inplanes, planes, 1, dilation, style=style,
                  with_cp=with_cp, normalize=normalize))
Kai Chen's avatar
Kai Chen committed
203
204
205
206

    return nn.Sequential(*layers)


Kai Chen's avatar
Kai Chen committed
207
208
class ResNet(nn.Module):
    """ResNet backbone.
Kai Chen's avatar
Kai Chen committed
209

Kai Chen's avatar
Kai Chen committed
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
    Args:
        depth (int): Depth of resnet, from {18, 34, 50, 101, 152}.
        num_stages (int): Resnet stages, normally 4.
        strides (Sequence[int]): Strides of the first block of each stage.
        dilations (Sequence[int]): Dilation of each stage.
        out_indices (Sequence[int]): Output from which stages.
        style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two
            layer is the 3x3 conv layer, otherwise the stride-two layer is
            the first 1x1 conv layer.
        frozen_stages (int): Stages to be frozen (all param fixed). -1 means
            not freezing any parameters.
        bn_eval (bool): Whether to set BN layers to eval mode, namely, freeze
            running stats (mean and var).
        bn_frozen (bool): Whether to freeze weight and bias of BN layers.
        with_cp (bool): Use checkpoint or not. Using checkpoint will save some
            memory while slowing down the training speed.
    """
Kai Chen's avatar
Kai Chen committed
227

Kai Chen's avatar
Kai Chen committed
228
229
230
231
232
233
234
    arch_settings = {
        18: (BasicBlock, (2, 2, 2, 2)),
        34: (BasicBlock, (3, 4, 6, 3)),
        50: (Bottleneck, (3, 4, 6, 3)),
        101: (Bottleneck, (3, 4, 23, 3)),
        152: (Bottleneck, (3, 8, 36, 3))
    }
Kai Chen's avatar
Kai Chen committed
235
236

    def __init__(self,
Kai Chen's avatar
Kai Chen committed
237
238
                 depth,
                 num_stages=4,
Kai Chen's avatar
Kai Chen committed
239
240
241
                 strides=(1, 2, 2, 2),
                 dilations=(1, 1, 1, 1),
                 out_indices=(0, 1, 2, 3),
242
                 style='pytorch',
ThangVu's avatar
ThangVu committed
243
                 frozen_stages=-1,
244
245
246
247
                 normalize=dict(
                     type='BN',
                     bn_eval=True,
                     bn_frozen=False),
Kai Chen's avatar
Kai Chen committed
248
                 with_cp=False):
Kai Chen's avatar
Kai Chen committed
249
        super(ResNet, self).__init__()
Kai Chen's avatar
Kai Chen committed
250
251
        if depth not in self.arch_settings:
            raise KeyError('invalid depth {} for resnet'.format(depth))
pangjm's avatar
pangjm committed
252
253
        self.depth = depth
        self.num_stages = num_stages
Kai Chen's avatar
Kai Chen committed
254
        assert num_stages >= 1 and num_stages <= 4
pangjm's avatar
pangjm committed
255
256
        self.strides = strides
        self.dilations = dilations
Kai Chen's avatar
Kai Chen committed
257
        assert len(strides) == len(dilations) == num_stages
Kai Chen's avatar
Kai Chen committed
258
        self.out_indices = out_indices
pangjm's avatar
pangjm committed
259
        assert max(out_indices) < num_stages
Kai Chen's avatar
Kai Chen committed
260
        self.style = style
ThangVu's avatar
ThangVu committed
261
        self.frozen_stages = frozen_stages
ThangVu's avatar
ThangVu committed
262
        self.with_cp = with_cp
Kai Chen's avatar
Kai Chen committed
263

264
265
266
267
268
        assert isinstance(normalize, dict) and 'type' in normalize
        assert normalize['type'] in ['BN', 'GN']
        if normalize['type'] == 'GN':
            assert 'num_groups' in normalize
        else:
ThangVu's avatar
ThangVu committed
269
            assert (set(['type', 'bn_eval', 'bn_frozen'])
270
271
272
273
274
                    == set(normalize))
        if normalize['type'] == 'BN':
            self.bn_eval = normalize['bn_eval']
            self.bn_frozen = normalize['bn_frozen']
        self.normalize = normalize
Kai Chen's avatar
Kai Chen committed
275

pangjm's avatar
pangjm committed
276
277
        self.block, stage_blocks = self.arch_settings[depth]
        self.stage_blocks = stage_blocks[:num_stages]
Kai Chen's avatar
Kai Chen committed
278
        self.inplanes = 64
pangjm's avatar
pangjm committed
279

Kai Chen's avatar
Kai Chen committed
280
281
        self.conv1 = nn.Conv2d(
            3, 64, kernel_size=7, stride=2, padding=3, bias=False)
282
283
284
        stem_norm = build_norm_layer(normalize, 64)
        self.stem_norm_name = 'gn1' if normalize['type'] == 'GN' else 'bn1'
        self.add_module(self.stem_norm_name, stem_norm)
Kai Chen's avatar
Kai Chen committed
285
286
287
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)

Kai Chen's avatar
Kai Chen committed
288
        self.res_layers = []
pangjm's avatar
pangjm committed
289
        for i, num_blocks in enumerate(self.stage_blocks):
Kai Chen's avatar
Kai Chen committed
290
291
292
293
            stride = strides[i]
            dilation = dilations[i]
            planes = 64 * 2**i
            res_layer = make_res_layer(
pangjm's avatar
pangjm committed
294
                self.block,
Kai Chen's avatar
Kai Chen committed
295
296
297
298
299
300
                self.inplanes,
                planes,
                num_blocks,
                stride=stride,
                dilation=dilation,
                style=self.style,
301
302
                with_cp=with_cp,
                normalize=normalize)
pangjm's avatar
pangjm committed
303
            self.inplanes = planes * self.block.expansion
Kai Chen's avatar
Kai Chen committed
304
            layer_name = 'layer{}'.format(i + 1)
305
            self.add_module(layer_name, res_layer)
Kai Chen's avatar
Kai Chen committed
306
307
            self.res_layers.append(layer_name)

pangjm's avatar
pangjm committed
308
309
        self.feat_dim = self.block.expansion * 64 * 2**(
            len(self.stage_blocks) - 1)
pangjm's avatar
pangjm committed
310

Kai Chen's avatar
Kai Chen committed
311
312
    def init_weights(self, pretrained=None):
        if isinstance(pretrained, str):
313
314
            logger = logging.getLogger()
            load_checkpoint(self, pretrained, strict=False, logger=logger)
Kai Chen's avatar
Kai Chen committed
315
316
317
        elif pretrained is None:
            for m in self.modules():
                if isinstance(m, nn.Conv2d):
Kai Chen's avatar
Kai Chen committed
318
                    kaiming_init(m)
Kai Chen's avatar
Kai Chen committed
319
                elif isinstance(m, nn.BatchNorm2d):
Kai Chen's avatar
Kai Chen committed
320
                    constant_init(m, 1)
321
322
323
324
325
326

            # zero init for last norm layer https://arxiv.org/abs/1706.02677
            for m in self.modules():
                if isinstance(m, Bottleneck) or isinstance(m, BasicBlock):
                    last_norm = getattr(m, m.norm_names[-1])
                    constant_init(last_norm, 0)
Kai Chen's avatar
Kai Chen committed
327
328
329
330
331
        else:
            raise TypeError('pretrained must be a str or None')

    def forward(self, x):
        x = self.conv1(x)
332
        x = getattr(self, self.stem_norm_name)(x)
Kai Chen's avatar
Kai Chen committed
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
        x = self.relu(x)
        x = self.maxpool(x)
        outs = []
        for i, layer_name in enumerate(self.res_layers):
            res_layer = getattr(self, layer_name)
            x = res_layer(x)
            if i in self.out_indices:
                outs.append(x)
        if len(outs) == 1:
            return outs[0]
        else:
            return tuple(outs)

    def train(self, mode=True):
        super(ResNet, self).train(mode)
ThangVu's avatar
ThangVu committed
348
349
350
351
352
353
354
355
356
357
        if self.normalize['type'] == 'BN' and self.bn_eval:
            for m in self.modules():
                if isinstance(m, nn.BatchNorm2d):
                    m.eval()
                    if self.bn_frozen:
                        for params in m.parameters():
                            params.requires_grad = False
        if mode and self.frozen_stages >= 0:
            for param in self.conv1.parameters():
                param.requires_grad = False
ThangVu's avatar
ThangVu committed
358

ThangVu's avatar
ThangVu committed
359
360
361
362
363
364
365
366
367
            stem_norm = getattr(self, self.stem_norm_name)
            stem_norm.eval()
            for param in stem_norm.parameters():
                param.requires_grad = False

            for i in range(1, self.frozen_stages + 1):
                mod = getattr(self, 'layer{}'.format(i))
                mod.eval()
                for param in mod.parameters():
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
                    param.requires_grad = False


class ResNetClassifier(ResNet):
    def __init__(self,
                 depth,
                 num_stages=4,
                 strides=(1, 2, 2, 2),
                 dilations=(1, 1, 1, 1),
                 out_indices=(0, 1, 2, 3),
                 style='pytorch',
                 normalize=dict(
                     type='BN',
                     frozen_stages=-1,
                     bn_eval=True,
                     bn_frozen=False),
                 with_cp=False,
                 num_classes=1000):
        super(ResNetClassifier, self).__init__(depth,
                                               num_stages=num_stages,
                                               strides=strides,
                                               dilations=dilations,
                                               out_indices=out_indices,
                                               style=style,
                                               normalize=normalize,
                                               with_cp=with_cp)
        _, self.stage_blocks = self.arch_settings[depth]
        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        expansion = 1 if depth == 18 else 4
        self.fc = nn.Linear(512 * expansion, num_classes)

        self.init_weights()

    # TODO can be removed after tested
    def load_caffe2_weight(self, cf_path):
        norm = 'gn' if self.normalize['type'] == 'GN' else 'bn'
        mapping = {}

        for layer, blocks_in_layer in enumerate(self.stage_blocks, 1):
            for blk in range(blocks_in_layer):
                cf_prefix = 'res%d_%d_' % (layer + 1, blk)
                py_prefix = 'layer%d.%d.' % (layer, blk)

                # conv branch
                for i, a in zip([1, 2, 3], ['a', 'b', 'c']):
                    cf_full = cf_prefix + 'branch2%s_' % a
                    mapping[py_prefix + 'conv%d.weight' % i] = cf_full + 'w'
                    mapping[py_prefix + norm + '%d.weight' % i] \
                        = cf_full + norm + '_s'
                    mapping[py_prefix + norm + '%d.bias' % i] \
                        = cf_full + norm + '_b'

            # downsample branch
            cf_full = 'res%d_0_branch1_' % (layer + 1)
            py_full = 'layer%d.0.downsample.' % layer
            mapping[py_full + '0.weight'] = cf_full + 'w'
            mapping[py_full + '1.weight'] = cf_full + norm + '_s'
            mapping[py_full + '1.bias'] = cf_full + norm + '_b'

        # stem layers and last fc layer
        if self.normalize['type'] == 'GN':
            mapping['conv1.weight'] = 'conv1_w'
            mapping['gn1.weight'] = 'conv1_gn_s'
            mapping['gn1.bias'] = 'conv1_gn_b'
            mapping['fc.weight'] = 'pred_w'
            mapping['fc.bias'] = 'pred_b'
        else:
            mapping['conv1.weight'] = 'conv1_w'
            mapping['bn1.weight'] = 'res_conv1_bn_s'
            mapping['bn1.bias'] = 'res_conv1_bn_b'
            mapping['fc.weight'] = 'fc1000_w'
            mapping['fc.bias'] = 'fc1000_b'

        # load state dict
        py_state = self.state_dict()
        with open(cf_path, 'rb') as f:
            cf_state = pickle.load(f, encoding='latin1')
            if 'blobs' in cf_state:
                cf_state = cf_state['blobs']
ThangVu's avatar
ThangVu committed
447
            for i, (py_k, cf_k) in enumerate(mapping.items(), 1):
ThangVu's avatar
ThangVu committed
448
449
                print('[{}/{}] Loading {} to {}'.format(
                    i, len(mapping), cf_k, py_k))
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
                assert py_k in py_state and cf_k in cf_state
                py_state[py_k] = torch.Tensor(cf_state[cf_k])
        self.load_state_dict(py_state)

    def forward(self, x):
        x = self.conv1(x)
        x = getattr(self, self.stem_norm_name)(x)
        x = self.relu(x)
        x = self.maxpool(x)
        for i, layer_name in enumerate(self.res_layers):
            res_layer = getattr(self, layer_name)
            x = res_layer(x)
        x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        x = self.fc(x)
        return x