test_processor_clip.py 8.17 KB
Newer Older
Suraj Patil's avatar
Suraj Patil committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

24
from transformers import CLIPTokenizer, CLIPTokenizerFast
Suraj Patil's avatar
Suraj Patil committed
25
26
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
27
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available
Suraj Patil's avatar
Suraj Patil committed
28
29
30
31
32


if is_vision_available():
    from PIL import Image

33
    from transformers import CLIPImageProcessor, CLIPProcessor
Suraj Patil's avatar
Suraj Patil committed
34
35
36
37
38
39
40


@require_vision
class CLIPProcessorTest(unittest.TestCase):
    def setUp(self):
        self.tmpdirname = tempfile.mkdtemp()

41
        vocab = ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]  # fmt: skip
Suraj Patil's avatar
Suraj Patil committed
42
43
44
45
46
47
48
49
50
51
52
        vocab_tokens = dict(zip(vocab, range(len(vocab))))
        merges = ["#version: 0.2", "l o", "lo w</w>", "e r</w>", ""]
        self.special_tokens_map = {"unk_token": "<unk>"}

        self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
        self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["merges_file"])
        with open(self.vocab_file, "w", encoding="utf-8") as fp:
            fp.write(json.dumps(vocab_tokens) + "\n")
        with open(self.merges_file, "w", encoding="utf-8") as fp:
            fp.write("\n".join(merges))

53
        image_processor_map = {
Suraj Patil's avatar
Suraj Patil committed
54
55
56
57
58
59
60
61
            "do_resize": True,
            "size": 20,
            "do_center_crop": True,
            "crop_size": 18,
            "do_normalize": True,
            "image_mean": [0.48145466, 0.4578275, 0.40821073],
            "image_std": [0.26862954, 0.26130258, 0.27577711],
        }
62
63
64
        self.image_processor_file = os.path.join(self.tmpdirname, IMAGE_PROCESSOR_NAME)
        with open(self.image_processor_file, "w", encoding="utf-8") as fp:
            json.dump(image_processor_map, fp)
Suraj Patil's avatar
Suraj Patil committed
65
66
67
68

    def get_tokenizer(self, **kwargs):
        return CLIPTokenizer.from_pretrained(self.tmpdirname, **kwargs)

69
70
71
    def get_rust_tokenizer(self, **kwargs):
        return CLIPTokenizerFast.from_pretrained(self.tmpdirname, **kwargs)

72
73
    def get_image_processor(self, **kwargs):
        return CLIPImageProcessor.from_pretrained(self.tmpdirname, **kwargs)
Suraj Patil's avatar
Suraj Patil committed
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

    def tearDown(self):
        shutil.rmtree(self.tmpdirname)

    def prepare_image_inputs(self):
        """This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True,
        or a list of PyTorch tensors if one specifies torchify=True.
        """

        image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)]

        image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs]

        return image_inputs

    def test_save_load_pretrained_default(self):
90
91
        tokenizer_slow = self.get_tokenizer()
        tokenizer_fast = self.get_rust_tokenizer()
92
        image_processor = self.get_image_processor()
Suraj Patil's avatar
Suraj Patil committed
93

94
        processor_slow = CLIPProcessor(tokenizer=tokenizer_slow, image_processor=image_processor)
95
96
        processor_slow.save_pretrained(self.tmpdirname)
        processor_slow = CLIPProcessor.from_pretrained(self.tmpdirname, use_fast=False)
Suraj Patil's avatar
Suraj Patil committed
97

98
        processor_fast = CLIPProcessor(tokenizer=tokenizer_fast, image_processor=image_processor)
99
100
        processor_fast.save_pretrained(self.tmpdirname)
        processor_fast = CLIPProcessor.from_pretrained(self.tmpdirname)
Suraj Patil's avatar
Suraj Patil committed
101

102
103
104
105
106
        self.assertEqual(processor_slow.tokenizer.get_vocab(), tokenizer_slow.get_vocab())
        self.assertEqual(processor_fast.tokenizer.get_vocab(), tokenizer_fast.get_vocab())
        self.assertEqual(tokenizer_slow.get_vocab(), tokenizer_fast.get_vocab())
        self.assertIsInstance(processor_slow.tokenizer, CLIPTokenizer)
        self.assertIsInstance(processor_fast.tokenizer, CLIPTokenizerFast)
Suraj Patil's avatar
Suraj Patil committed
107

108
109
110
111
        self.assertEqual(processor_slow.image_processor.to_json_string(), image_processor.to_json_string())
        self.assertEqual(processor_fast.image_processor.to_json_string(), image_processor.to_json_string())
        self.assertIsInstance(processor_slow.image_processor, CLIPImageProcessor)
        self.assertIsInstance(processor_fast.image_processor, CLIPImageProcessor)
Suraj Patil's avatar
Suraj Patil committed
112
113

    def test_save_load_pretrained_additional_features(self):
114
        processor = CLIPProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
Suraj Patil's avatar
Suraj Patil committed
115
116
117
        processor.save_pretrained(self.tmpdirname)

        tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
118
        image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
Suraj Patil's avatar
Suraj Patil committed
119
120
121
122
123
124

        processor = CLIPProcessor.from_pretrained(
            self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
        )

        self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
125
        self.assertIsInstance(processor.tokenizer, CLIPTokenizerFast)
Suraj Patil's avatar
Suraj Patil committed
126

127
128
        self.assertEqual(processor.image_processor.to_json_string(), image_processor_add_kwargs.to_json_string())
        self.assertIsInstance(processor.image_processor, CLIPImageProcessor)
Suraj Patil's avatar
Suraj Patil committed
129

130
131
    def test_image_processor(self):
        image_processor = self.get_image_processor()
Suraj Patil's avatar
Suraj Patil committed
132
133
        tokenizer = self.get_tokenizer()

134
        processor = CLIPProcessor(tokenizer=tokenizer, image_processor=image_processor)
Suraj Patil's avatar
Suraj Patil committed
135
136
137

        image_input = self.prepare_image_inputs()

138
        input_image_proc = image_processor(image_input, return_tensors="np")
Suraj Patil's avatar
Suraj Patil committed
139
140
        input_processor = processor(images=image_input, return_tensors="np")

141
142
        for key in input_image_proc.keys():
            self.assertAlmostEqual(input_image_proc[key].sum(), input_processor[key].sum(), delta=1e-2)
Suraj Patil's avatar
Suraj Patil committed
143
144

    def test_tokenizer(self):
145
        image_processor = self.get_image_processor()
Suraj Patil's avatar
Suraj Patil committed
146
147
        tokenizer = self.get_tokenizer()

148
        processor = CLIPProcessor(tokenizer=tokenizer, image_processor=image_processor)
Suraj Patil's avatar
Suraj Patil committed
149
150
151
152
153
154
155
156
157
158
159

        input_str = "lower newer"

        encoded_processor = processor(text=input_str)

        encoded_tok = tokenizer(input_str)

        for key in encoded_tok.keys():
            self.assertListEqual(encoded_tok[key], encoded_processor[key])

    def test_processor(self):
160
        image_processor = self.get_image_processor()
Suraj Patil's avatar
Suraj Patil committed
161
162
        tokenizer = self.get_tokenizer()

163
        processor = CLIPProcessor(tokenizer=tokenizer, image_processor=image_processor)
Suraj Patil's avatar
Suraj Patil committed
164
165
166
167
168
169
170
171
172
173
174
175
176

        input_str = "lower newer"
        image_input = self.prepare_image_inputs()

        inputs = processor(text=input_str, images=image_input)

        self.assertListEqual(list(inputs.keys()), ["input_ids", "attention_mask", "pixel_values"])

        # test if it raises when no input is passed
        with pytest.raises(ValueError):
            processor()

    def test_tokenizer_decode(self):
177
        image_processor = self.get_image_processor()
Suraj Patil's avatar
Suraj Patil committed
178
179
        tokenizer = self.get_tokenizer()

180
        processor = CLIPProcessor(tokenizer=tokenizer, image_processor=image_processor)
Suraj Patil's avatar
Suraj Patil committed
181
182
183
184
185
186
187

        predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

        decoded_processor = processor.batch_decode(predicted_ids)
        decoded_tok = tokenizer.batch_decode(predicted_ids)

        self.assertListEqual(decoded_tok, decoded_processor)
188
189

    def test_model_input_names(self):
190
        image_processor = self.get_image_processor()
191
192
        tokenizer = self.get_tokenizer()

193
        processor = CLIPProcessor(tokenizer=tokenizer, image_processor=image_processor)
194
195
196
197
198
199
200

        input_str = "lower newer"
        image_input = self.prepare_image_inputs()

        inputs = processor(text=input_str, images=image_input)

        self.assertListEqual(list(inputs.keys()), processor.model_input_names)