test_modeling_swin.py 20.2 KB
Newer Older
novice's avatar
novice committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2022 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Arthur's avatar
Arthur committed
15
"""Testing suite for the PyTorch Swin model."""
novice's avatar
novice committed
16

NielsRogge's avatar
NielsRogge committed
17
import collections
novice's avatar
novice committed
18
19
20
21
import unittest

from transformers import SwinConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
22
from transformers.utils import cached_property, is_torch_available, is_vision_available
novice's avatar
novice committed
23

24
from ...test_backbone_common import BackboneTesterMixin
Yih-Dar's avatar
Yih-Dar committed
25
from ...test_configuration_common import ConfigTester
26
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
27
from ...test_pipeline_mixin import PipelineTesterMixin
novice's avatar
novice committed
28
29
30
31
32
33


if is_torch_available():
    import torch
    from torch import nn

NielsRogge's avatar
NielsRogge committed
34
    from transformers import SwinBackbone, SwinForImageClassification, SwinForMaskedImageModeling, SwinModel
35

novice's avatar
novice committed
36
37
38
39

if is_vision_available():
    from PIL import Image

40
    from transformers import AutoImageProcessor
novice's avatar
novice committed
41
42
43
44
45
46
47
48
49
50
51


class SwinModelTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        image_size=32,
        patch_size=2,
        num_channels=3,
        embed_dim=16,
52
53
        depths=[1, 2, 1],
        num_heads=[2, 2, 4],
novice's avatar
novice committed
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
        window_size=2,
        mlp_ratio=2.0,
        qkv_bias=True,
        hidden_dropout_prob=0.0,
        attention_probs_dropout_prob=0.0,
        drop_path_rate=0.1,
        hidden_act="gelu",
        use_absolute_embeddings=False,
        patch_norm=True,
        initializer_range=0.02,
        layer_norm_eps=1e-5,
        is_training=True,
        scope=None,
        use_labels=True,
        type_sequence_label_size=10,
69
        encoder_stride=8,
NielsRogge's avatar
NielsRogge committed
70
        out_features=["stage1", "stage2"],
71
        out_indices=[1, 2],
novice's avatar
novice committed
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.image_size = image_size
        self.patch_size = patch_size
        self.num_channels = num_channels
        self.embed_dim = embed_dim
        self.depths = depths
        self.num_heads = num_heads
        self.window_size = window_size
        self.mlp_ratio = mlp_ratio
        self.qkv_bias = qkv_bias
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.drop_path_rate = drop_path_rate
        self.hidden_act = hidden_act
        self.use_absolute_embeddings = use_absolute_embeddings
        self.patch_norm = patch_norm
        self.layer_norm_eps = layer_norm_eps
        self.initializer_range = initializer_range
        self.is_training = is_training
        self.scope = scope
        self.use_labels = use_labels
        self.type_sequence_label_size = type_sequence_label_size
NielsRogge's avatar
NielsRogge committed
96
        self.encoder_stride = encoder_stride
NielsRogge's avatar
NielsRogge committed
97
        self.out_features = out_features
98
        self.out_indices = out_indices
novice's avatar
novice committed
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

    def prepare_config_and_inputs(self):
        pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

        labels = None
        if self.use_labels:
            labels = ids_tensor([self.batch_size], self.type_sequence_label_size)

        config = self.get_config()

        return config, pixel_values, labels

    def get_config(self):
        return SwinConfig(
            image_size=self.image_size,
            patch_size=self.patch_size,
            num_channels=self.num_channels,
            embed_dim=self.embed_dim,
            depths=self.depths,
            num_heads=self.num_heads,
            window_size=self.window_size,
            mlp_ratio=self.mlp_ratio,
            qkv_bias=self.qkv_bias,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            drop_path_rate=self.drop_path_rate,
            hidden_act=self.hidden_act,
            use_absolute_embeddings=self.use_absolute_embeddings,
            path_norm=self.patch_norm,
            layer_norm_eps=self.layer_norm_eps,
            initializer_range=self.initializer_range,
NielsRogge's avatar
NielsRogge committed
130
            encoder_stride=self.encoder_stride,
NielsRogge's avatar
NielsRogge committed
131
            out_features=self.out_features,
132
            out_indices=self.out_indices,
novice's avatar
novice committed
133
134
135
136
137
138
139
140
        )

    def create_and_check_model(self, config, pixel_values, labels):
        model = SwinModel(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)

141
        expected_seq_len = ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths) - 1))
142
        expected_dim = int(config.embed_dim * 2 ** (len(config.depths) - 1))
novice's avatar
novice committed
143

144
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, expected_seq_len, expected_dim))
novice's avatar
novice committed
145

NielsRogge's avatar
NielsRogge committed
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
    def create_and_check_backbone(self, config, pixel_values, labels):
        model = SwinBackbone(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)

        # verify hidden states
        self.parent.assertEqual(len(result.feature_maps), len(config.out_features))
        self.parent.assertListEqual(list(result.feature_maps[0].shape), [self.batch_size, model.channels[0], 16, 16])

        # verify channels
        self.parent.assertEqual(len(model.channels), len(config.out_features))

        # verify backbone works with out_features=None
        config.out_features = None
        model = SwinBackbone(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)

        # verify feature maps
        self.parent.assertEqual(len(result.feature_maps), 1)
        self.parent.assertListEqual(list(result.feature_maps[0].shape), [self.batch_size, model.channels[-1], 4, 4])

        # verify channels
        self.parent.assertEqual(len(model.channels), 1)

NielsRogge's avatar
NielsRogge committed
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
    def create_and_check_for_masked_image_modeling(self, config, pixel_values, labels):
        model = SwinForMaskedImageModeling(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)
        self.parent.assertEqual(
            result.logits.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size)
        )

        # test greyscale images
        config.num_channels = 1
        model = SwinForMaskedImageModeling(config)
        model.to(torch_device)
        model.eval()

        pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
        result = model(pixel_values)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, 1, self.image_size, self.image_size))

novice's avatar
novice committed
192
193
194
195
196
197
198
199
    def create_and_check_for_image_classification(self, config, pixel_values, labels):
        config.num_labels = self.type_sequence_label_size
        model = SwinForImageClassification(config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values, labels=labels)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.type_sequence_label_size))

NielsRogge's avatar
NielsRogge committed
200
201
202
203
204
205
206
207
208
209
        # test greyscale images
        config.num_channels = 1
        model = SwinForImageClassification(config)
        model.to(torch_device)
        model.eval()

        pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
        result = model(pixel_values)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.type_sequence_label_size))

novice's avatar
novice committed
210
211
212
213
214
215
216
217
218
219
220
221
    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
            config,
            pixel_values,
            labels,
        ) = config_and_inputs
        inputs_dict = {"pixel_values": pixel_values}
        return config, inputs_dict


@require_torch
222
class SwinModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
novice's avatar
novice committed
223
224
225
    all_model_classes = (
        (
            SwinModel,
NielsRogge's avatar
NielsRogge committed
226
            SwinBackbone,
novice's avatar
novice committed
227
            SwinForImageClassification,
NielsRogge's avatar
NielsRogge committed
228
            SwinForMaskedImageModeling,
novice's avatar
novice committed
229
230
231
232
        )
        if is_torch_available()
        else ()
    )
233
    pipeline_model_mapping = (
234
        {"image-feature-extraction": SwinModel, "image-classification": SwinForImageClassification}
235
236
237
        if is_torch_available()
        else {}
    )
238
    fx_compatible = True
novice's avatar
novice committed
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263

    test_pruning = False
    test_resize_embeddings = False
    test_head_masking = False

    def setUp(self):
        self.model_tester = SwinModelTester(self)
        self.config_tester = ConfigTester(self, config_class=SwinConfig, embed_dim=37)

    def test_config(self):
        self.create_and_test_config_common_properties()
        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()
        self.config_tester.check_config_arguments_init()

    def create_and_test_config_common_properties(self):
        return

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

264
265
266
267
268
    # TODO: check if this works again for PyTorch 2.x.y
    @unittest.skip(reason="Got `CUDA error: misaligned address` with PyTorch 2.0.0.")
    def test_multi_gpu_data_parallel_forward(self):
        pass

269
270
271
    def test_training_gradient_checkpointing(self):
        super().test_training_gradient_checkpointing()

NielsRogge's avatar
NielsRogge committed
272
273
274
275
    def test_backbone(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_backbone(*config_and_inputs)

NielsRogge's avatar
NielsRogge committed
276
277
278
279
280
281
282
283
    def test_for_masked_image_modeling(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_image_modeling(*config_and_inputs)

    def test_for_image_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*config_and_inputs)

NielsRogge's avatar
NielsRogge committed
284
    @unittest.skip(reason="Swin does not use inputs_embeds")
novice's avatar
novice committed
285
    def test_inputs_embeds(self):
NielsRogge's avatar
NielsRogge committed
286
287
288
289
        pass

    @unittest.skip(reason="Swin Transformer does not use feedforward chunking")
    def test_feed_forward_chunking(self):
novice's avatar
novice committed
290
291
        pass

292
    def test_model_get_set_embeddings(self):
novice's avatar
novice committed
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            self.assertIsInstance(model.get_input_embeddings(), (nn.Module))
            x = model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(x, nn.Linear))

    def test_attention_outputs(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        for model_class in self.all_model_classes:
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = False
            config.return_dict = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
314
315
316
            attentions = outputs.attentions
            expected_num_attentions = len(self.model_tester.depths)
            self.assertEqual(len(attentions), expected_num_attentions)
novice's avatar
novice committed
317
318
319
320

            # check that output_attentions also work using config
            del inputs_dict["output_attentions"]
            config.output_attentions = True
321
            window_size_squared = config.window_size**2
novice's avatar
novice committed
322
323
324
325
326
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
327
328
329
330
331
332
333
            attentions = outputs.attentions
            self.assertEqual(len(attentions), expected_num_attentions)

            self.assertListEqual(
                list(attentions[0].shape[-3:]),
                [self.model_tester.num_heads[0], window_size_squared, window_size_squared],
            )
novice's avatar
novice committed
334
335
336
337
338
339
340
341
342
343
344
            out_len = len(outputs)

            # Check attention is always last and order is fine
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

NielsRogge's avatar
NielsRogge committed
345
346
            # also another +1 for reshaped_hidden_states
            added_hidden_states = 1 if model_class.__name__ == "SwinBackbone" else 2
novice's avatar
novice committed
347
348
            self.assertEqual(out_len + added_hidden_states, len(outputs))

349
            self_attentions = outputs.attentions
novice's avatar
novice committed
350

351
352
353
354
355
356
            self.assertEqual(len(self_attentions), expected_num_attentions)

            self.assertListEqual(
                list(self_attentions[0].shape[-3:]),
                [self.model_tester.num_heads[0], window_size_squared, window_size_squared],
            )
novice's avatar
novice committed
357

358
359
360
361
    def check_hidden_states_output(self, inputs_dict, config, model_class, image_size):
        model = model_class(config)
        model.to(torch_device)
        model.eval()
novice's avatar
novice committed
362

363
364
        with torch.no_grad():
            outputs = model(**self._prepare_for_class(inputs_dict, model_class))
novice's avatar
novice committed
365

366
        hidden_states = outputs.hidden_states
novice's avatar
novice committed
367

368
369
370
371
        expected_num_layers = getattr(
            self.model_tester, "expected_num_hidden_layers", len(self.model_tester.depths) + 1
        )
        self.assertEqual(len(hidden_states), expected_num_layers)
novice's avatar
novice committed
372

373
        # Swin has a different seq_length
NielsRogge's avatar
NielsRogge committed
374
375
376
377
378
        patch_size = (
            config.patch_size
            if isinstance(config.patch_size, collections.abc.Iterable)
            else (config.patch_size, config.patch_size)
        )
379

380
        num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
novice's avatar
novice committed
381

382
383
384
385
        self.assertListEqual(
            list(hidden_states[0].shape[-2:]),
            [num_patches, self.model_tester.embed_dim],
        )
novice's avatar
novice committed
386

NielsRogge's avatar
NielsRogge committed
387
388
389
        if not model_class.__name__ == "SwinBackbone":
            reshaped_hidden_states = outputs.reshaped_hidden_states
            self.assertEqual(len(reshaped_hidden_states), expected_num_layers)
390

NielsRogge's avatar
NielsRogge committed
391
392
393
394
395
396
397
398
            batch_size, num_channels, height, width = reshaped_hidden_states[0].shape
            reshaped_hidden_states = (
                reshaped_hidden_states[0].view(batch_size, num_channels, height * width).permute(0, 2, 1)
            )
            self.assertListEqual(
                list(reshaped_hidden_states.shape[-2:]),
                [num_patches, self.model_tester.embed_dim],
            )
399

400
    def test_hidden_states_output(self):
novice's avatar
novice committed
401
402
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

NielsRogge's avatar
NielsRogge committed
403
404
405
406
407
        image_size = (
            self.model_tester.image_size
            if isinstance(self.model_tester.image_size, collections.abc.Iterable)
            else (self.model_tester.image_size, self.model_tester.image_size)
        )
408

novice's avatar
novice committed
409
410
        for model_class in self.all_model_classes:
            inputs_dict["output_hidden_states"] = True
411
            self.check_hidden_states_output(inputs_dict, config, model_class, image_size)
novice's avatar
novice committed
412
413
414
415
416

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True

417
418
419
420
421
422
            self.check_hidden_states_output(inputs_dict, config, model_class, image_size)

    def test_hidden_states_output_with_padding(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.patch_size = 3

NielsRogge's avatar
NielsRogge committed
423
424
425
426
427
428
429
430
431
432
        image_size = (
            self.model_tester.image_size
            if isinstance(self.model_tester.image_size, collections.abc.Iterable)
            else (self.model_tester.image_size, self.model_tester.image_size)
        )
        patch_size = (
            config.patch_size
            if isinstance(config.patch_size, collections.abc.Iterable)
            else (config.patch_size, config.patch_size)
        )
433
434
435
436
437
438
439
440
441
442
443
444

        padded_height = image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
        padded_width = image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

        for model_class in self.all_model_classes:
            inputs_dict["output_hidden_states"] = True
            self.check_hidden_states_output(inputs_dict, config, model_class, (padded_height, padded_width))

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True
            self.check_hidden_states_output(inputs_dict, config, model_class, (padded_height, padded_width))
novice's avatar
novice committed
445
446
447

    @slow
    def test_model_from_pretrained(self):
448
449
450
        model_name = "microsoft/swin-tiny-patch4-window7-224"
        model = SwinModel.from_pretrained(model_name)
        self.assertIsNotNone(model)
novice's avatar
novice committed
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470

    def test_initialization(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        configs_no_init = _config_zero_init(config)
        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            for name, param in model.named_parameters():
                if "embeddings" not in name and param.requires_grad:
                    self.assertIn(
                        ((param.data.mean() * 1e9).round() / 1e9).item(),
                        [0.0, 1.0],
                        msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                    )


@require_vision
@require_torch
class SwinModelIntegrationTest(unittest.TestCase):
    @cached_property
471
    def default_image_processor(self):
novice's avatar
novice committed
472
        return (
473
            AutoImageProcessor.from_pretrained("microsoft/swin-tiny-patch4-window7-224")
novice's avatar
novice committed
474
475
476
477
478
479
480
            if is_vision_available()
            else None
        )

    @slow
    def test_inference_image_classification_head(self):
        model = SwinForImageClassification.from_pretrained("microsoft/swin-tiny-patch4-window7-224").to(torch_device)
481
        image_processor = self.default_image_processor
novice's avatar
novice committed
482
483

        image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")
484
        inputs = image_processor(images=image, return_tensors="pt").to(torch_device)
novice's avatar
novice committed
485
486
487
488
489
490
491
492

        # forward pass
        with torch.no_grad():
            outputs = model(**inputs)

        # verify the logits
        expected_shape = torch.Size((1, 1000))
        self.assertEqual(outputs.logits.shape, expected_shape)
493
        expected_slice = torch.tensor([-0.0948, -0.6454, -0.0921]).to(torch_device)
novice's avatar
novice committed
494
        self.assertTrue(torch.allclose(outputs.logits[0, :3], expected_slice, atol=1e-4))
495

496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
    @slow
    def test_inference_interpolate_pos_encoding(self):
        # Swin models have an `interpolate_pos_encoding` argument in their forward method,
        # allowing to interpolate the pre-trained position embeddings in order to use
        # the model on higher resolutions.
        model = SwinModel.from_pretrained("microsoft/swin-tiny-patch4-window7-224").to(torch_device)

        image_processor = self.default_image_processor
        image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")
        inputs = image_processor(images=image, size={"height": 481, "width": 481}, return_tensors="pt")
        pixel_values = inputs.pixel_values.to(torch_device)

        # forward pass
        with torch.no_grad():
            outputs = model(pixel_values, interpolate_pos_encoding=True)

        # verify the logits
        expected_shape = torch.Size((1, 256, 768))
        self.assertEqual(outputs.last_hidden_state.shape, expected_shape)

516
517
518
519
520
521
522
523

@require_torch
class SwinBackboneTest(unittest.TestCase, BackboneTesterMixin):
    all_model_classes = (SwinBackbone,) if is_torch_available() else ()
    config_class = SwinConfig

    def setUp(self):
        self.model_tester = SwinModelTester(self)