test_modeling_deit.py 16.3 KB
Newer Older
NielsRogge's avatar
NielsRogge committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Arthur's avatar
Arthur committed
15
"""Testing suite for the PyTorch DeiT model."""
NielsRogge's avatar
NielsRogge committed
16
17

import unittest
18
import warnings
NielsRogge's avatar
NielsRogge committed
19

20
from transformers import DeiTConfig
21
22
23
from transformers.testing_utils import (
    require_accelerate,
    require_torch,
24
25
    require_torch_accelerator,
    require_torch_fp16,
26
27
28
29
    require_vision,
    slow,
    torch_device,
)
30
from transformers.utils import cached_property, is_torch_available, is_vision_available
NielsRogge's avatar
NielsRogge committed
31

Yih-Dar's avatar
Yih-Dar committed
32
33
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
34
from ...test_pipeline_mixin import PipelineTesterMixin
NielsRogge's avatar
NielsRogge committed
35
36
37
38


if is_torch_available():
    import torch
39
    from torch import nn
NielsRogge's avatar
NielsRogge committed
40
41
42
43

    from transformers import (
        DeiTForImageClassification,
        DeiTForImageClassificationWithTeacher,
NielsRogge's avatar
NielsRogge committed
44
        DeiTForMaskedImageModeling,
NielsRogge's avatar
NielsRogge committed
45
46
        DeiTModel,
    )
47
48
49
50
51
    from transformers.models.auto.modeling_auto import (
        MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
        MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
        MODEL_MAPPING_NAMES,
    )
NielsRogge's avatar
NielsRogge committed
52
53
54
55
56


if is_vision_available():
    from PIL import Image

57
    from transformers import DeiTImageProcessor
NielsRogge's avatar
NielsRogge committed
58
59
60
61
62
63
64
65
66
67
68
69
70


class DeiTModelTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        image_size=30,
        patch_size=2,
        num_channels=3,
        is_training=True,
        use_labels=True,
        hidden_size=32,
71
        num_hidden_layers=2,
NielsRogge's avatar
NielsRogge committed
72
73
74
75
76
77
78
79
80
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        type_sequence_label_size=10,
        initializer_range=0.02,
        num_labels=3,
        scope=None,
NielsRogge's avatar
NielsRogge committed
81
        encoder_stride=2,
82
83
        mask_ratio=0.5,
        attn_implementation="eager",
NielsRogge's avatar
NielsRogge committed
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.image_size = image_size
        self.patch_size = patch_size
        self.num_channels = num_channels
        self.is_training = is_training
        self.use_labels = use_labels
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.type_sequence_label_size = type_sequence_label_size
        self.initializer_range = initializer_range
        self.scope = scope
NielsRogge's avatar
NielsRogge committed
102
        self.encoder_stride = encoder_stride
103
        self.attn_implementation = attn_implementation
NielsRogge's avatar
NielsRogge committed
104

NielsRogge's avatar
NielsRogge committed
105
        # in DeiT, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distilation tokens)
106
        num_patches = (image_size // patch_size) ** 2
NielsRogge's avatar
NielsRogge committed
107
        self.seq_length = num_patches + 2
108
109
110
        self.mask_ratio = mask_ratio
        self.num_masks = int(mask_ratio * self.seq_length)
        self.mask_length = num_patches
111

NielsRogge's avatar
NielsRogge committed
112
113
114
115
116
117
118
    def prepare_config_and_inputs(self):
        pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

        labels = None
        if self.use_labels:
            labels = ids_tensor([self.batch_size], self.type_sequence_label_size)

119
120
121
122
123
124
        config = self.get_config()

        return config, pixel_values, labels

    def get_config(self):
        return DeiTConfig(
NielsRogge's avatar
NielsRogge committed
125
126
127
128
129
130
131
132
133
134
135
136
            image_size=self.image_size,
            patch_size=self.patch_size,
            num_channels=self.num_channels,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            is_decoder=False,
            initializer_range=self.initializer_range,
NielsRogge's avatar
NielsRogge committed
137
            encoder_stride=self.encoder_stride,
138
            attn_implementation=self.attn_implementation,
NielsRogge's avatar
NielsRogge committed
139
140
141
142
143
144
145
        )

    def create_and_check_model(self, config, pixel_values, labels):
        model = DeiTModel(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)
NielsRogge's avatar
NielsRogge committed
146
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
NielsRogge's avatar
NielsRogge committed
147

NielsRogge's avatar
NielsRogge committed
148
149
150
151
152
153
    def create_and_check_for_masked_image_modeling(self, config, pixel_values, labels):
        model = DeiTForMaskedImageModeling(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)
        self.parent.assertEqual(
154
            result.reconstruction.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size)
NielsRogge's avatar
NielsRogge committed
155
156
157
158
159
160
161
162
163
164
        )

        # test greyscale images
        config.num_channels = 1
        model = DeiTForMaskedImageModeling(config)
        model.to(torch_device)
        model.eval()

        pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
        result = model(pixel_values)
165
        self.parent.assertEqual(result.reconstruction.shape, (self.batch_size, 1, self.image_size, self.image_size))
NielsRogge's avatar
NielsRogge committed
166

NielsRogge's avatar
NielsRogge committed
167
168
169
170
171
172
173
174
    def create_and_check_for_image_classification(self, config, pixel_values, labels):
        config.num_labels = self.type_sequence_label_size
        model = DeiTForImageClassification(config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values, labels=labels)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.type_sequence_label_size))

NielsRogge's avatar
NielsRogge committed
175
176
177
178
179
180
181
182
183
184
        # test greyscale images
        config.num_channels = 1
        model = DeiTForImageClassification(config)
        model.to(torch_device)
        model.eval()

        pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
        result = model(pixel_values, labels=labels)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.type_sequence_label_size))

NielsRogge's avatar
NielsRogge committed
185
186
187
188
189
190
191
192
193
194
195
196
    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
            config,
            pixel_values,
            labels,
        ) = config_and_inputs
        inputs_dict = {"pixel_values": pixel_values}
        return config, inputs_dict


@require_torch
197
class DeiTModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
NielsRogge's avatar
NielsRogge committed
198
199
200
201
202
203
204
205
206
207
    """
    Here we also overwrite some of the tests of test_modeling_common.py, as DeiT does not use input_ids, inputs_embeds,
    attention_mask and seq_length.
    """

    all_model_classes = (
        (
            DeiTModel,
            DeiTForImageClassification,
            DeiTForImageClassificationWithTeacher,
NielsRogge's avatar
NielsRogge committed
208
            DeiTForMaskedImageModeling,
NielsRogge's avatar
NielsRogge committed
209
210
211
212
        )
        if is_torch_available()
        else ()
    )
213
214
    pipeline_model_mapping = (
        {
215
            "image-feature-extraction": DeiTModel,
216
217
218
219
220
            "image-classification": (DeiTForImageClassification, DeiTForImageClassificationWithTeacher),
        }
        if is_torch_available()
        else {}
    )
NielsRogge's avatar
NielsRogge committed
221
222
223
224
225
226
227
228
229
230
231
232

    test_pruning = False
    test_resize_embeddings = False
    test_head_masking = False

    def setUp(self):
        self.model_tester = DeiTModelTester(self)
        self.config_tester = ConfigTester(self, config_class=DeiTConfig, has_text_modality=False, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

NielsRogge's avatar
NielsRogge committed
233
    @unittest.skip(reason="DeiT does not use inputs_embeds")
NielsRogge's avatar
NielsRogge committed
234
235
236
237
238
239
240
241
    def test_inputs_embeds(self):
        pass

    def test_model_common_attributes(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
242
            self.assertIsInstance(model.get_input_embeddings(), (nn.Module))
NielsRogge's avatar
NielsRogge committed
243
            x = model.get_output_embeddings()
244
            self.assertTrue(x is None or isinstance(x, nn.Linear))
NielsRogge's avatar
NielsRogge committed
245
246
247
248
249

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

NielsRogge's avatar
NielsRogge committed
250
251
252
253
    def test_for_masked_image_modeling(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_image_modeling(*config_and_inputs)

NielsRogge's avatar
NielsRogge committed
254
255
256
    def test_for_image_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*config_and_inputs)
NielsRogge's avatar
NielsRogge committed
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277

    # special case for DeiTForImageClassificationWithTeacher model
    def _prepare_for_class(self, inputs_dict, model_class, return_labels=False):
        inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels)

        if return_labels:
            if model_class.__name__ == "DeiTForImageClassificationWithTeacher":
                del inputs_dict["labels"]

        return inputs_dict

    def test_training(self):
        if not self.model_tester.is_training:
            return

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        for model_class in self.all_model_classes:
            # DeiTForImageClassificationWithTeacher supports inference-only
            if (
278
                model_class.__name__ in MODEL_MAPPING_NAMES.values()
NielsRogge's avatar
NielsRogge committed
279
280
281
282
283
284
285
286
287
288
                or model_class.__name__ == "DeiTForImageClassificationWithTeacher"
            ):
                continue
            model = model_class(config)
            model.to(torch_device)
            model.train()
            inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            loss = model(**inputs).loss
            loss.backward()

289
290
291
292
293
294
295
296
297
    def test_training_gradient_checkpointing(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        if not self.model_tester.is_training:
            return

        config.use_cache = False
        config.return_dict = True

        for model_class in self.all_model_classes:
298
            if model_class.__name__ in MODEL_MAPPING_NAMES.values() or not model_class.supports_gradient_checkpointing:
299
300
301
302
303
                continue
            # DeiTForImageClassificationWithTeacher supports inference-only
            if model_class.__name__ == "DeiTForImageClassificationWithTeacher":
                continue
            model = model_class(config)
304
            model.gradient_checkpointing_enable()
305
306
307
308
309
310
            model.to(torch_device)
            model.train()
            inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            loss = model(**inputs).loss
            loss.backward()

311
312
313
314
315
316
317
318
319
320
321
322
    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass

323
324
325
326
327
328
329
330
331
332
333
    def test_problem_types(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        problem_types = [
            {"title": "multi_label_classification", "num_labels": 2, "dtype": torch.float},
            {"title": "single_label_classification", "num_labels": 1, "dtype": torch.long},
            {"title": "regression", "num_labels": 1, "dtype": torch.float},
        ]

        for model_class in self.all_model_classes:
            if (
334
                model_class.__name__
335
                not in [
336
337
                    *MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES.values(),
                    *MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES.values(),
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
                ]
                or model_class.__name__ == "DeiTForImageClassificationWithTeacher"
            ):
                continue

            for problem_type in problem_types:
                with self.subTest(msg=f"Testing {model_class} with {problem_type['title']}"):
                    config.problem_type = problem_type["title"]
                    config.num_labels = problem_type["num_labels"]

                    model = model_class(config)
                    model.to(torch_device)
                    model.train()

                    inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)

                    if problem_type["num_labels"] > 1:
                        inputs["labels"] = inputs["labels"].unsqueeze(1).repeat(1, problem_type["num_labels"])

                    inputs["labels"] = inputs["labels"].to(problem_type["dtype"])

                    # This tests that we do not trigger the warning form PyTorch "Using a target size that is different
                    # to the input size. This will likely lead to incorrect results due to broadcasting. Please ensure
                    # they have the same size." which is a symptom something in wrong for the regression problem.
                    # See https://github.com/huggingface/transformers/issues/11780
                    with warnings.catch_warnings(record=True) as warning_list:
                        loss = model(**inputs).loss
                    for w in warning_list:
                        if "Using a target size that is different to the input size" in str(w.message):
                            raise ValueError(
                                f"Something is going wrong in the regression problem: intercepted {w.message}"
                            )

                    loss.backward()

NielsRogge's avatar
NielsRogge committed
373
374
    @slow
    def test_model_from_pretrained(self):
375
376
377
        model_name = "facebook/deit-base-distilled-patch16-224"
        model = DeiTModel.from_pretrained(model_name)
        self.assertIsNotNone(model)
NielsRogge's avatar
NielsRogge committed
378
379
380
381


# We will verify our results on an image of cute cats
def prepare_img():
NielsRogge's avatar
NielsRogge committed
382
    image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")
NielsRogge's avatar
NielsRogge committed
383
384
385
    return image


386
@require_torch
NielsRogge's avatar
NielsRogge committed
387
388
389
@require_vision
class DeiTModelIntegrationTest(unittest.TestCase):
    @cached_property
390
    def default_image_processor(self):
NielsRogge's avatar
NielsRogge committed
391
        return (
392
            DeiTImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
NielsRogge's avatar
NielsRogge committed
393
394
395
396
397
398
399
400
401
402
            if is_vision_available()
            else None
        )

    @slow
    def test_inference_image_classification_head(self):
        model = DeiTForImageClassificationWithTeacher.from_pretrained("facebook/deit-base-distilled-patch16-224").to(
            torch_device
        )

403
        image_processor = self.default_image_processor
NielsRogge's avatar
NielsRogge committed
404
        image = prepare_img()
405
        inputs = image_processor(images=image, return_tensors="pt").to(torch_device)
NielsRogge's avatar
NielsRogge committed
406
407

        # forward pass
408
409
        with torch.no_grad():
            outputs = model(**inputs)
NielsRogge's avatar
NielsRogge committed
410
411
412
413
414
415
416
417

        # verify the logits
        expected_shape = torch.Size((1, 1000))
        self.assertEqual(outputs.logits.shape, expected_shape)

        expected_slice = torch.tensor([-1.0266, 0.1912, -1.2861]).to(torch_device)

        self.assertTrue(torch.allclose(outputs.logits[0, :3], expected_slice, atol=1e-4))
418
419
420

    @slow
    @require_accelerate
421
422
    @require_torch_accelerator
    @require_torch_fp16
423
424
425
426
427
428
429
    def test_inference_fp16(self):
        r"""
        A small test to make sure that inference work in half precision without any problem.
        """
        model = DeiTModel.from_pretrained(
            "facebook/deit-base-distilled-patch16-224", torch_dtype=torch.float16, device_map="auto"
        )
430
        image_processor = self.default_image_processor
431
432

        image = prepare_img()
433
        inputs = image_processor(images=image, return_tensors="pt")
434
435
436
437
438
        pixel_values = inputs.pixel_values.to(torch_device)

        # forward pass to make sure inference works in fp16
        with torch.no_grad():
            _ = model(pixel_values)