test_tokenization_t5.py 8.88 KB
Newer Older
thomwolf's avatar
thomwolf committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2018 Google T5 Authors and HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Aymeric Augustin's avatar
Aymeric Augustin committed
15

thomwolf's avatar
thomwolf committed
16

17
import unittest
thomwolf's avatar
thomwolf committed
18

19
from transformers import SPIECE_UNDERLINE, BatchEncoding, T5Tokenizer, T5TokenizerFast
20
21
from transformers.file_utils import cached_property, is_torch_available
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers
thomwolf's avatar
thomwolf committed
22

23
from .test_tokenization_common import TokenizerTesterMixin
thomwolf's avatar
thomwolf committed
24

Aymeric Augustin's avatar
Aymeric Augustin committed
25

26
SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")
27

28
FRAMEWORK = "pt" if is_torch_available() else "tf"
29

thomwolf's avatar
thomwolf committed
30

31
32
@require_sentencepiece
@require_tokenizers
33
class T5TokenizationTest(TokenizerTesterMixin, unittest.TestCase):
thomwolf's avatar
thomwolf committed
34
35

    tokenizer_class = T5Tokenizer
36
37
    rust_tokenizer_class = T5TokenizerFast
    test_rust_tokenizer = True
thomwolf's avatar
thomwolf committed
38
39

    def setUp(self):
Julien Chaumond's avatar
Julien Chaumond committed
40
        super().setUp()
thomwolf's avatar
thomwolf committed
41
42

        # We have a SentencePiece fixture for testing
43
        tokenizer = T5Tokenizer(SAMPLE_VOCAB)
thomwolf's avatar
thomwolf committed
44
45
46
        tokenizer.save_pretrained(self.tmpdirname)

    def test_full_tokenizer(self):
47
        tokenizer = T5Tokenizer(SAMPLE_VOCAB)
thomwolf's avatar
thomwolf committed
48

49
50
        tokens = tokenizer.tokenize("This is a test")
        self.assertListEqual(tokens, ["鈻乀his", "鈻乮s", "鈻乤", "鈻乼", "est"])
thomwolf's avatar
thomwolf committed
51

52
        self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [285, 46, 10, 170, 382])
thomwolf's avatar
thomwolf committed
53

54
        tokens = tokenizer.tokenize("I was born in 92000, and this is fals茅.")
thomwolf's avatar
thomwolf committed
55
        self.assertListEqual(
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
            tokens,
            [
                SPIECE_UNDERLINE + "I",
                SPIECE_UNDERLINE + "was",
                SPIECE_UNDERLINE + "b",
                "or",
                "n",
                SPIECE_UNDERLINE + "in",
                SPIECE_UNDERLINE + "",
                "9",
                "2",
                "0",
                "0",
                "0",
                ",",
                SPIECE_UNDERLINE + "and",
                SPIECE_UNDERLINE + "this",
                SPIECE_UNDERLINE + "is",
                SPIECE_UNDERLINE + "f",
                "al",
                "s",
                "茅",
                ".",
            ],
        )
        ids = tokenizer.convert_tokens_to_ids(tokens)
        self.assertListEqual(ids, [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4])
thomwolf's avatar
thomwolf committed
83
84

        back_tokens = tokenizer.convert_ids_to_tokens(ids)
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
        self.assertListEqual(
            back_tokens,
            [
                SPIECE_UNDERLINE + "I",
                SPIECE_UNDERLINE + "was",
                SPIECE_UNDERLINE + "b",
                "or",
                "n",
                SPIECE_UNDERLINE + "in",
                SPIECE_UNDERLINE + "",
                "<unk>",
                "2",
                "0",
                "0",
                "0",
                ",",
                SPIECE_UNDERLINE + "and",
                SPIECE_UNDERLINE + "this",
                SPIECE_UNDERLINE + "is",
                SPIECE_UNDERLINE + "f",
                "al",
                "s",
                "<unk>",
                ".",
            ],
        )
111

112
113
114
115
    @cached_property
    def t5_base_tokenizer(self):
        return T5Tokenizer.from_pretrained("t5-base")

116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
    @cached_property
    def t5_base_tokenizer_fast(self):
        return T5TokenizerFast.from_pretrained("t5-base")

    def get_tokenizer(self, **kwargs) -> T5Tokenizer:
        return self.tokenizer_class.from_pretrained(self.tmpdirname, pad_token=None, **kwargs)

    def get_rust_tokenizer(self, **kwargs) -> T5TokenizerFast:
        return self.rust_tokenizer_class.from_pretrained(self.tmpdirname, pad_token=None, **kwargs)

    def test_rust_and_python_full_tokenizers(self):
        if not self.test_rust_tokenizer:
            return

        tokenizer = self.get_tokenizer()
        rust_tokenizer = self.get_rust_tokenizer()

        sequence = "I was born in 92000, and this is fals茅."

        tokens = tokenizer.tokenize(sequence)
        rust_tokens = rust_tokenizer.tokenize(sequence)
        self.assertListEqual(tokens, rust_tokens)

        ids = tokenizer.encode(sequence, add_special_tokens=False)
        rust_ids = rust_tokenizer.encode(sequence, add_special_tokens=False)
        self.assertListEqual(ids, rust_ids)

        rust_tokenizer = self.get_rust_tokenizer()
        ids = tokenizer.encode(sequence)
        rust_ids = rust_tokenizer.encode(sequence)
        self.assertListEqual(ids, rust_ids)

148
149
150
151
152
153
    def test_eos_treatment(self):
        tokenizer = self.t5_base_tokenizer
        batch_with_eos_added = tokenizer(["hi</s>", "I went to the gym</s>", "</s>"])
        batch_without_eos_added = tokenizer(["hi", "I went to the gym", ""])
        self.assertListEqual(batch_with_eos_added["input_ids"], batch_without_eos_added["input_ids"])

154
    def test_prepare_batch(self):
155
        tokenizer = self.t5_base_tokenizer
156
157
        src_text = ["A long paragraph for summarization.", "Another paragraph for summarization."]
        expected_src_tokens = [71, 307, 8986, 21, 4505, 1635, 1707, 5, tokenizer.eos_token_id]
158
        batch = tokenizer(src_text, padding=True, return_tensors=FRAMEWORK)
159
160
161
        self.assertIsInstance(batch, BatchEncoding)
        result = list(batch.input_ids.numpy()[0])
        self.assertListEqual(expected_src_tokens, result)
162

163
164
        self.assertEqual((2, 9), batch.input_ids.shape)
        self.assertEqual((2, 9), batch.attention_mask.shape)
165

166
    def test_empty_target_text(self):
167
        tokenizer = self.t5_base_tokenizer
168
        src_text = ["A long paragraph for summarization.", "Another paragraph for summarization."]
169
        batch = tokenizer(src_text, padding=True, return_tensors=FRAMEWORK)
170
171
172
173
174
175
        # check if input_ids are returned and no decoder_input_ids
        self.assertIn("input_ids", batch)
        self.assertIn("attention_mask", batch)
        self.assertNotIn("decoder_input_ids", batch)
        self.assertNotIn("decoder_attention_mask", batch)

176
    def test_max_length(self):
177
        tokenizer = self.t5_base_tokenizer
178
179
180
181
        tgt_text = [
            "Summary of the text.",
            "Another summary.",
        ]
182
183
184
185
186
        with tokenizer.as_target_tokenizer():
            targets = tokenizer(
                tgt_text, max_length=32, padding="max_length", truncation=True, return_tensors=FRAMEWORK
            )
        self.assertEqual(32, targets["input_ids"].shape[1])
187
188

    def test_outputs_not_longer_than_maxlen(self):
189
        tokenizer = self.t5_base_tokenizer
190

191
192
        batch = tokenizer(
            ["I am a small frog" * 1000, "I am a small frog"], padding=True, truncation=True, return_tensors=FRAMEWORK
193
194
195
196
197
        )
        self.assertIsInstance(batch, BatchEncoding)
        self.assertEqual(batch.input_ids.shape, (2, 512))

    def test_eos_in_input(self):
198
        tokenizer = self.t5_base_tokenizer
199
        src_text = ["A long paragraph for summarization. </s>"]
200
        tgt_text = ["Summary of the text. </s>"]
201
        expected_src_tokens = [71, 307, 8986, 21, 4505, 1635, 1707, 5, 1]
202
        expected_tgt_tokens = [20698, 13, 8, 1499, 5, 1]
203

204
205
206
        batch = tokenizer(src_text)
        with tokenizer.as_target_tokenizer():
            targets = tokenizer(tgt_text)
207

208
209
        self.assertEqual(expected_src_tokens, batch["input_ids"][0])
        self.assertEqual(expected_tgt_tokens, targets["input_ids"][0])
210

211
212
213
214
215
216
217
218
219
220
221
222
223
224
    def test_token_type_ids(self):
        src_text_1 = ["A first paragraph for summarization."]
        src_text_2 = ["A second paragraph for summarization."]

        fast_token_type_ids = self.t5_base_tokenizer_fast(
            src_text_1, src_text_2, add_special_tokens=True, return_token_type_ids=True
        ).token_type_ids
        slow_token_type_ids = self.t5_base_tokenizer(
            src_text_1, src_text_2, add_special_tokens=True, return_token_type_ids=True
        ).token_type_ids

        self.assertEqual(slow_token_type_ids, fast_token_type_ids)
        self.assertEqual(len(slow_token_type_ids[0]), 18)

225
226
227
228
229
230
231
232
233
234
235
236
237
238
    def test_fast_and_slow_same_result(self):
        src_text = "<pad> Today is <unk> nice day </s>"
        tgt_ids = [0, 1960, 19, 2, 1245, 239, 1]
        tgt_text = "<pad> Today is<unk> nice day</s>"

        fast_ids = self.t5_base_tokenizer_fast(src_text, add_special_tokens=False).input_ids
        slow_ids = self.t5_base_tokenizer(src_text, add_special_tokens=False).input_ids
        self.assertEqual(tgt_ids, fast_ids)
        self.assertEqual(tgt_ids, slow_ids)

        fast_text = self.t5_base_tokenizer_fast.decode(fast_ids)
        slow_text = self.t5_base_tokenizer.decode(fast_ids)
        self.assertEqual(tgt_text, fast_text)
        self.assertEqual(tgt_text, slow_text)