test_image_processing_flava.py 16 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# coding=utf-8
# Copyright 2022 Meta Platforms authors and HuggingFace Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import random
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

24
from ...test_image_processing_common import ImageProcessingTestMixin, prepare_image_inputs
25
26
27
28
29
30


if is_torch_available():
    import torch

if is_vision_available():
amyeroberts's avatar
amyeroberts committed
31
    import PIL
32

33
    from transformers import FlavaImageProcessor
34
    from transformers.image_utils import PILImageResampling
amyeroberts's avatar
amyeroberts committed
35
    from transformers.models.flava.image_processing_flava import (
36
37
38
39
40
41
42
43
44
        FLAVA_CODEBOOK_MEAN,
        FLAVA_CODEBOOK_STD,
        FLAVA_IMAGE_MEAN,
        FLAVA_IMAGE_STD,
    )
else:
    FLAVA_IMAGE_MEAN = FLAVA_IMAGE_STD = FLAVA_CODEBOOK_MEAN = FLAVA_CODEBOOK_STD = None


45
class FlavaImageProcessingTester(unittest.TestCase):
46
47
48
49
50
51
52
53
    def __init__(
        self,
        parent,
        batch_size=7,
        num_channels=3,
        min_resolution=30,
        max_resolution=400,
        do_resize=True,
amyeroberts's avatar
amyeroberts committed
54
        size=None,
55
        do_center_crop=True,
amyeroberts's avatar
amyeroberts committed
56
        crop_size=None,
57
        resample=None,
amyeroberts's avatar
amyeroberts committed
58
59
        do_rescale=True,
        rescale_factor=1 / 255,
60
61
62
63
64
65
66
67
68
69
        do_normalize=True,
        image_mean=FLAVA_IMAGE_MEAN,
        image_std=FLAVA_IMAGE_STD,
        input_size_patches=14,
        total_mask_patches=75,
        mask_group_max_patches=None,
        mask_group_min_patches=16,
        mask_group_min_aspect_ratio=0.3,
        mask_group_max_aspect_ratio=None,
        codebook_do_resize=True,
amyeroberts's avatar
amyeroberts committed
70
        codebook_size=None,
71
72
        codebook_resample=None,
        codebook_do_center_crop=True,
amyeroberts's avatar
amyeroberts committed
73
        codebook_crop_size=None,
74
75
76
77
78
        codebook_do_map_pixels=True,
        codebook_do_normalize=True,
        codebook_image_mean=FLAVA_CODEBOOK_MEAN,
        codebook_image_std=FLAVA_CODEBOOK_STD,
    ):
79
        super().__init__()
amyeroberts's avatar
amyeroberts committed
80
81
82
83
84
        size = size if size is not None else {"height": 224, "width": 224}
        crop_size = crop_size if crop_size is not None else {"height": 224, "width": 224}
        codebook_size = codebook_size if codebook_size is not None else {"height": 112, "width": 112}
        codebook_crop_size = codebook_crop_size if codebook_crop_size is not None else {"height": 112, "width": 112}

85
86
87
88
        self.parent = parent
        self.batch_size = batch_size
        self.num_channels = num_channels
        self.do_resize = do_resize
amyeroberts's avatar
amyeroberts committed
89
90
        self.do_rescale = do_rescale
        self.rescale_factor = rescale_factor
91
92
93
        self.min_resolution = min_resolution
        self.max_resolution = max_resolution
        self.size = size
94
        self.resample = resample if resample is not None else PILImageResampling.BICUBIC
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
        self.do_normalize = do_normalize
        self.image_mean = image_mean
        self.image_std = image_std
        self.do_center_crop = do_center_crop
        self.crop_size = crop_size

        self.input_size_patches = input_size_patches
        self.total_mask_patches = total_mask_patches
        self.mask_group_max_patches = mask_group_max_patches
        self.mask_group_min_patches = mask_group_min_patches
        self.mask_group_min_aspect_ratio = mask_group_min_aspect_ratio
        self.mask_group_max_aspect_ratio = mask_group_max_aspect_ratio

        self.codebook_do_resize = codebook_do_resize
        self.codebook_size = codebook_size
110
        self.codebook_resample = codebook_resample if codebook_resample is not None else PILImageResampling.LANCZOS
111
112
113
114
115
116
117
        self.codebook_do_center_crop = codebook_do_center_crop
        self.codebook_crop_size = codebook_crop_size
        self.codebook_do_map_pixels = codebook_do_map_pixels
        self.codebook_do_normalize = codebook_do_normalize
        self.codebook_image_mean = codebook_image_mean
        self.codebook_image_std = codebook_image_std

118
    def prepare_image_processor_dict(self):
119
120
121
122
123
124
125
        return {
            "image_mean": self.image_mean,
            "image_std": self.image_std,
            "do_normalize": self.do_normalize,
            "do_resize": self.do_resize,
            "size": self.size,
            "resample": self.resample,
amyeroberts's avatar
amyeroberts committed
126
127
            "do_rescale": self.do_rescale,
            "rescale_factor": self.rescale_factor,
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
            "do_center_crop": self.do_center_crop,
            "crop_size": self.crop_size,
            "input_size_patches": self.input_size_patches,
            "total_mask_patches": self.total_mask_patches,
            "mask_group_max_patches": self.mask_group_max_patches,
            "mask_group_min_patches": self.mask_group_min_patches,
            "mask_group_min_aspect_ratio": self.mask_group_min_aspect_ratio,
            "mask_group_max_aspect_ratio": self.mask_group_min_aspect_ratio,
            "codebook_do_resize": self.codebook_do_resize,
            "codebook_size": self.codebook_size,
            "codebook_resample": self.codebook_resample,
            "codebook_do_center_crop": self.codebook_do_center_crop,
            "codebook_crop_size": self.codebook_crop_size,
            "codebook_do_map_pixels": self.codebook_do_map_pixels,
            "codebook_do_normalize": self.codebook_do_normalize,
            "codebook_image_mean": self.codebook_image_mean,
            "codebook_image_std": self.codebook_image_std,
        }

    def get_expected_image_size(self):
amyeroberts's avatar
amyeroberts committed
148
        return (self.size["height"], self.size["width"])
149
150
151
152
153
154
155
156
157

    def get_expected_mask_size(self):
        return (
            (self.input_size_patches, self.input_size_patches)
            if not isinstance(self.input_size_patches, tuple)
            else self.input_size_patches
        )

    def get_expected_codebook_image_size(self):
amyeroberts's avatar
amyeroberts committed
158
        return (self.codebook_size["height"], self.codebook_size["width"])
159

160
161
162
163
164
165
166
167
168
169
170
    def prepare_image_inputs(self, equal_resolution=False, numpify=False, torchify=False):
        return prepare_image_inputs(
            batch_size=self.batch_size,
            num_channels=self.num_channels,
            min_resolution=self.min_resolution,
            max_resolution=self.max_resolution,
            equal_resolution=equal_resolution,
            numpify=numpify,
            torchify=torchify,
        )

171
172
173

@require_torch
@require_vision
174
class FlavaImageProcessingTest(ImageProcessingTestMixin, unittest.TestCase):
175
    image_processing_class = FlavaImageProcessor if is_vision_available() else None
176
177
178
    maxDiff = None

    def setUp(self):
amyeroberts's avatar
amyeroberts committed
179
        super().setUp()
180
        self.image_processor_tester = FlavaImageProcessingTester(self)
181
182

    @property
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
    def image_processor_dict(self):
        return self.image_processor_tester.prepare_image_processor_dict()

    def test_image_processor_properties(self):
        image_processing = self.image_processing_class(**self.image_processor_dict)
        self.assertTrue(hasattr(image_processing, "image_mean"))
        self.assertTrue(hasattr(image_processing, "image_std"))
        self.assertTrue(hasattr(image_processing, "do_normalize"))
        self.assertTrue(hasattr(image_processing, "do_resize"))
        self.assertTrue(hasattr(image_processing, "resample"))
        self.assertTrue(hasattr(image_processing, "crop_size"))
        self.assertTrue(hasattr(image_processing, "do_center_crop"))
        self.assertTrue(hasattr(image_processing, "do_rescale"))
        self.assertTrue(hasattr(image_processing, "rescale_factor"))
        self.assertTrue(hasattr(image_processing, "masking_generator"))
        self.assertTrue(hasattr(image_processing, "codebook_do_resize"))
        self.assertTrue(hasattr(image_processing, "codebook_size"))
        self.assertTrue(hasattr(image_processing, "codebook_resample"))
        self.assertTrue(hasattr(image_processing, "codebook_do_center_crop"))
        self.assertTrue(hasattr(image_processing, "codebook_crop_size"))
        self.assertTrue(hasattr(image_processing, "codebook_do_map_pixels"))
        self.assertTrue(hasattr(image_processing, "codebook_do_normalize"))
        self.assertTrue(hasattr(image_processing, "codebook_image_mean"))
        self.assertTrue(hasattr(image_processing, "codebook_image_std"))

    def test_image_processor_from_dict_with_kwargs(self):
        image_processor = self.image_processing_class.from_dict(self.image_processor_dict)
        self.assertEqual(image_processor.size, {"height": 224, "width": 224})
        self.assertEqual(image_processor.crop_size, {"height": 224, "width": 224})
        self.assertEqual(image_processor.codebook_size, {"height": 112, "width": 112})
        self.assertEqual(image_processor.codebook_crop_size, {"height": 112, "width": 112})

        image_processor = self.image_processing_class.from_dict(
            self.image_processor_dict, size=42, crop_size=84, codebook_size=33, codebook_crop_size=66
217
        )
218
219
220
221
        self.assertEqual(image_processor.size, {"height": 42, "width": 42})
        self.assertEqual(image_processor.crop_size, {"height": 84, "width": 84})
        self.assertEqual(image_processor.codebook_size, {"height": 33, "width": 33})
        self.assertEqual(image_processor.codebook_crop_size, {"height": 66, "width": 66})
222

223
    def test_call_pil(self):
224
225
        # Initialize image_processing
        image_processing = self.image_processing_class(**self.image_processor_dict)
226
        # create random PIL images
227
        image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False)
228
        for image in image_inputs:
amyeroberts's avatar
amyeroberts committed
229
            self.assertIsInstance(image, PIL.Image.Image)
230
231

        # Test not batched input
232
        encoded_images = image_processing(image_inputs[0], return_tensors="pt")
233
234
235
236

        # Test no bool masked pos
        self.assertFalse("bool_masked_pos" in encoded_images)

237
        expected_height, expected_width = self.image_processor_tester.get_expected_image_size()
238
239
240

        self.assertEqual(
            encoded_images.pixel_values.shape,
241
            (1, self.image_processor_tester.num_channels, expected_height, expected_width),
242
243
244
        )

        # Test batched
245
246
        encoded_images = image_processing(image_inputs, return_tensors="pt")
        expected_height, expected_width = self.image_processor_tester.get_expected_image_size()
247
248
249
250
251
252
253

        # Test no bool masked pos
        self.assertFalse("bool_masked_pos" in encoded_images)

        self.assertEqual(
            encoded_images.pixel_values.shape,
            (
254
255
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
256
257
258
259
260
261
                expected_height,
                expected_width,
            ),
        )

    def _test_call_framework(self, instance_class, prepare_kwargs):
262
263
        # Initialize image_processing
        image_processing = self.image_processing_class(**self.image_processor_dict)
264
        # create random tensors
265
        image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, **prepare_kwargs)
266
267
268
269
        for image in image_inputs:
            self.assertIsInstance(image, instance_class)

        # Test not batched input
270
        encoded_images = image_processing(image_inputs[0], return_tensors="pt")
271

272
        expected_height, expected_width = self.image_processor_tester.get_expected_image_size()
273
274
        self.assertEqual(
            encoded_images.pixel_values.shape,
275
            (1, self.image_processor_tester.num_channels, expected_height, expected_width),
276
277
        )

278
        encoded_images = image_processing(image_inputs, return_image_mask=True, return_tensors="pt")
279

280
        expected_height, expected_width = self.image_processor_tester.get_expected_image_size()
281
282
283
        self.assertEqual(
            encoded_images.pixel_values.shape,
            (
284
285
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
286
287
288
289
290
                expected_height,
                expected_width,
            ),
        )

291
        expected_height, expected_width = self.image_processor_tester.get_expected_mask_size()
292
293
294
        self.assertEqual(
            encoded_images.bool_masked_pos.shape,
            (
295
                self.image_processor_tester.batch_size,
296
297
298
299
300
301
                expected_height,
                expected_width,
            ),
        )

        # Test batched
302
        encoded_images = image_processing(image_inputs, return_tensors="pt").pixel_values
303

304
        expected_height, expected_width = self.image_processor_tester.get_expected_image_size()
305
306
307
        self.assertEqual(
            encoded_images.shape,
            (
308
309
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
310
311
312
313
314
315
                expected_height,
                expected_width,
            ),
        )

        # Test masking
316
        encoded_images = image_processing(image_inputs, return_image_mask=True, return_tensors="pt")
317

318
        expected_height, expected_width = self.image_processor_tester.get_expected_image_size()
319
320
321
        self.assertEqual(
            encoded_images.pixel_values.shape,
            (
322
323
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
324
325
326
327
328
                expected_height,
                expected_width,
            ),
        )

329
        expected_height, expected_width = self.image_processor_tester.get_expected_mask_size()
330
331
332
        self.assertEqual(
            encoded_images.bool_masked_pos.shape,
            (
333
                self.image_processor_tester.batch_size,
334
335
336
337
338
339
340
341
                expected_height,
                expected_width,
            ),
        )

    def test_call_numpy(self):
        self._test_call_framework(np.ndarray, prepare_kwargs={"numpify": True})

amyeroberts's avatar
amyeroberts committed
342
343
344
345
346
    def test_call_numpy_4_channels(self):
        self.image_processing_class.num_channels = 4
        self._test_call_framework(np.ndarray, prepare_kwargs={"numpify": True})
        self.image_processing_class.num_channels = 3

347
348
349
350
    def test_call_pytorch(self):
        self._test_call_framework(torch.Tensor, prepare_kwargs={"torchify": True})

    def test_masking(self):
351
        # Initialize image_processing
352
        random.seed(1234)
353
        image_processing = self.image_processing_class(**self.image_processor_dict)
354
        image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False, torchify=True)
355
356

        # Test not batched input
357
        encoded_images = image_processing(image_inputs[0], return_image_mask=True, return_tensors="pt")
358
359
360
        self.assertEqual(encoded_images.bool_masked_pos.sum().item(), 75)

    def test_codebook_pixels(self):
361
362
        # Initialize image_processing
        image_processing = self.image_processing_class(**self.image_processor_dict)
363
        # create random PIL images
364
        image_inputs = self.image_processor_tester.prepare_image_inputs(equal_resolution=False)
365
        for image in image_inputs:
amyeroberts's avatar
amyeroberts committed
366
            self.assertIsInstance(image, PIL.Image.Image)
367
368

        # Test not batched input
369
370
        encoded_images = image_processing(image_inputs[0], return_codebook_pixels=True, return_tensors="pt")
        expected_height, expected_width = self.image_processor_tester.get_expected_codebook_image_size()
371
372
        self.assertEqual(
            encoded_images.codebook_pixel_values.shape,
373
            (1, self.image_processor_tester.num_channels, expected_height, expected_width),
374
375
376
        )

        # Test batched
377
378
        encoded_images = image_processing(image_inputs, return_codebook_pixels=True, return_tensors="pt")
        expected_height, expected_width = self.image_processor_tester.get_expected_codebook_image_size()
379
380
381
        self.assertEqual(
            encoded_images.codebook_pixel_values.shape,
            (
382
383
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
384
385
386
387
                expected_height,
                expected_width,
            ),
        )