test_modeling_xlm.py 14.3 KB
Newer Older
thomwolf's avatar
thomwolf committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2018 The Google AI Language Team Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Aymeric Augustin's avatar
Aymeric Augustin committed
15

thomwolf's avatar
thomwolf committed
16

17
18
import unittest

19
from transformers import is_torch_available
20
from transformers.testing_utils import require_torch, slow, torch_device
thomwolf's avatar
thomwolf committed
21

22
from .test_configuration_common import ConfigTester
23
from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
Aymeric Augustin's avatar
Aymeric Augustin committed
24
25


26
if is_torch_available():
27
    import torch
28

29
30
    from transformers import (
        XLMConfig,
31
        XLMForMultipleChoice,
32
33
        XLMForQuestionAnswering,
        XLMForQuestionAnsweringSimple,
34
35
36
37
        XLMForSequenceClassification,
        XLMForTokenClassification,
        XLMModel,
        XLMWithLMHeadModel,
38
    )
39
    from transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST
thomwolf's avatar
thomwolf committed
40
41


42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
class XLMModelTester:
    def __init__(
        self, parent,
    ):
        self.parent = parent
        self.batch_size = 13
        self.seq_length = 7
        self.is_training = True
        self.use_input_lengths = True
        self.use_token_type_ids = True
        self.use_labels = True
        self.gelu_activation = True
        self.sinusoidal_embeddings = False
        self.causal = False
        self.asm = False
        self.n_langs = 2
        self.vocab_size = 99
        self.n_special = 0
        self.hidden_size = 32
        self.num_hidden_layers = 5
        self.num_attention_heads = 4
        self.hidden_dropout_prob = 0.1
        self.attention_probs_dropout_prob = 0.1
        self.max_position_embeddings = 512
        self.type_sequence_label_size = 2
        self.initializer_range = 0.02
68
        self.num_labels = 2
69
70
71
72
73
74
75
76
        self.num_choices = 4
        self.summary_type = "last"
        self.use_proj = True
        self.scope = None
        self.bos_token_id = 0

    def prepare_config_and_inputs(self):
        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
77
        input_mask = random_attention_mask([self.batch_size, self.seq_length])
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

        input_lengths = None
        if self.use_input_lengths:
            input_lengths = (
                ids_tensor([self.batch_size], vocab_size=2) + self.seq_length - 2
            )  # small variation of seq_length

        token_type_ids = None
        if self.use_token_type_ids:
            token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.n_langs)

        sequence_labels = None
        token_labels = None
        is_impossible_labels = None
        if self.use_labels:
            sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
            token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
            is_impossible_labels = ids_tensor([self.batch_size], 2).float()
96
            choice_labels = ids_tensor([self.batch_size], self.num_choices)
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

        config = XLMConfig(
            vocab_size=self.vocab_size,
            n_special=self.n_special,
            emb_dim=self.hidden_size,
            n_layers=self.num_hidden_layers,
            n_heads=self.num_attention_heads,
            dropout=self.hidden_dropout_prob,
            attention_dropout=self.attention_probs_dropout_prob,
            gelu_activation=self.gelu_activation,
            sinusoidal_embeddings=self.sinusoidal_embeddings,
            asm=self.asm,
            causal=self.causal,
            n_langs=self.n_langs,
            max_position_embeddings=self.max_position_embeddings,
            initializer_range=self.initializer_range,
            summary_type=self.summary_type,
            use_proj=self.use_proj,
115
            num_labels=self.num_labels,
116
            bos_token_id=self.bos_token_id,
Sylvain Gugger's avatar
Sylvain Gugger committed
117
            return_dict=True,
118
        )
thomwolf's avatar
thomwolf committed
119

120
        return (
121
122
123
124
125
126
127
            config,
            input_ids,
            token_type_ids,
            input_lengths,
            sequence_labels,
            token_labels,
            is_impossible_labels,
128
            choice_labels,
129
            input_mask,
130
131
132
133
134
135
136
137
138
139
140
        )

    def create_and_check_xlm_model(
        self,
        config,
        input_ids,
        token_type_ids,
        input_lengths,
        sequence_labels,
        token_labels,
        is_impossible_labels,
141
        choice_labels,
142
143
144
145
146
        input_mask,
    ):
        model = XLMModel(config=config)
        model.to(torch_device)
        model.eval()
Sylvain Gugger's avatar
Sylvain Gugger committed
147
148
149
        result = model(input_ids, lengths=input_lengths, langs=token_type_ids)
        result = model(input_ids, langs=token_type_ids)
        result = model(input_ids)
Stas Bekman's avatar
Stas Bekman committed
150
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
151
152
153
154
155
156
157
158
159
160

    def create_and_check_xlm_lm_head(
        self,
        config,
        input_ids,
        token_type_ids,
        input_lengths,
        sequence_labels,
        token_labels,
        is_impossible_labels,
161
        choice_labels,
162
163
164
165
166
167
        input_mask,
    ):
        model = XLMWithLMHeadModel(config)
        model.to(torch_device)
        model.eval()

Sylvain Gugger's avatar
Sylvain Gugger committed
168
        result = model(input_ids, token_type_ids=token_type_ids, labels=token_labels)
Stas Bekman's avatar
Stas Bekman committed
169
170
        self.parent.assertEqual(result.loss.shape, ())
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
171
172
173
174
175
176
177
178
179
180

    def create_and_check_xlm_simple_qa(
        self,
        config,
        input_ids,
        token_type_ids,
        input_lengths,
        sequence_labels,
        token_labels,
        is_impossible_labels,
181
        choice_labels,
182
183
184
185
186
187
188
189
190
        input_mask,
    ):
        model = XLMForQuestionAnsweringSimple(config)
        model.to(torch_device)
        model.eval()

        outputs = model(input_ids)

        outputs = model(input_ids, start_positions=sequence_labels, end_positions=sequence_labels)
Sylvain Gugger's avatar
Sylvain Gugger committed
191
        result = outputs
Stas Bekman's avatar
Stas Bekman committed
192
193
        self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
        self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))
194
195
196
197
198
199
200
201
202
203

    def create_and_check_xlm_qa(
        self,
        config,
        input_ids,
        token_type_ids,
        input_lengths,
        sequence_labels,
        token_labels,
        is_impossible_labels,
204
        choice_labels,
205
206
207
208
209
210
        input_mask,
    ):
        model = XLMForQuestionAnswering(config)
        model.to(torch_device)
        model.eval()

Sylvain Gugger's avatar
Sylvain Gugger committed
211
        result = model(input_ids)
212

Sylvain Gugger's avatar
Sylvain Gugger committed
213
        result_with_labels = model(
214
            input_ids,
215
216
217
218
219
220
            start_positions=sequence_labels,
            end_positions=sequence_labels,
            cls_index=sequence_labels,
            is_impossible=is_impossible_labels,
            p_mask=input_mask,
        )
thomwolf's avatar
thomwolf committed
221

Sylvain Gugger's avatar
Sylvain Gugger committed
222
        result_with_labels = model(
223
224
225
226
227
228
            input_ids,
            start_positions=sequence_labels,
            end_positions=sequence_labels,
            cls_index=sequence_labels,
            is_impossible=is_impossible_labels,
        )
thomwolf's avatar
thomwolf committed
229

Sylvain Gugger's avatar
Sylvain Gugger committed
230
        (total_loss,) = result_with_labels.to_tuple()
thomwolf's avatar
thomwolf committed
231

Sylvain Gugger's avatar
Sylvain Gugger committed
232
        result_with_labels = model(input_ids, start_positions=sequence_labels, end_positions=sequence_labels)
233

Sylvain Gugger's avatar
Sylvain Gugger committed
234
        (total_loss,) = result_with_labels.to_tuple()
235

Stas Bekman's avatar
Stas Bekman committed
236
237
238
239
240
        self.parent.assertEqual(result_with_labels.loss.shape, ())
        self.parent.assertEqual(result.start_top_log_probs.shape, (self.batch_size, model.config.start_n_top))
        self.parent.assertEqual(result.start_top_index.shape, (self.batch_size, model.config.start_n_top))
        self.parent.assertEqual(
            result.end_top_log_probs.shape, (self.batch_size, model.config.start_n_top * model.config.end_n_top)
241
        )
Stas Bekman's avatar
Stas Bekman committed
242
243
        self.parent.assertEqual(
            result.end_top_index.shape, (self.batch_size, model.config.start_n_top * model.config.end_n_top)
244
        )
Stas Bekman's avatar
Stas Bekman committed
245
        self.parent.assertEqual(result.cls_logits.shape, (self.batch_size,))
246
247
248
249
250
251
252
253
254
255

    def create_and_check_xlm_sequence_classif(
        self,
        config,
        input_ids,
        token_type_ids,
        input_lengths,
        sequence_labels,
        token_labels,
        is_impossible_labels,
256
        choice_labels,
257
258
259
260
261
262
        input_mask,
    ):
        model = XLMForSequenceClassification(config)
        model.to(torch_device)
        model.eval()

Sylvain Gugger's avatar
Sylvain Gugger committed
263
264
        result = model(input_ids)
        result = model(input_ids, labels=sequence_labels)
Stas Bekman's avatar
Stas Bekman committed
265
266
        self.parent.assertEqual(result.loss.shape, ())
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.type_sequence_label_size))
267

268
    def create_and_check_xlm_token_classif(
269
270
271
272
273
274
275
276
        self,
        config,
        input_ids,
        token_type_ids,
        input_lengths,
        sequence_labels,
        token_labels,
        is_impossible_labels,
277
        choice_labels,
278
279
280
281
282
283
284
        input_mask,
    ):
        config.num_labels = self.num_labels
        model = XLMForTokenClassification(config)
        model.to(torch_device)
        model.eval()

Sylvain Gugger's avatar
Sylvain Gugger committed
285
        result = model(input_ids, attention_mask=input_mask, labels=token_labels)
Stas Bekman's avatar
Stas Bekman committed
286
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))
287

288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
    def create_and_check_xlm_for_multiple_choice(
        self,
        config,
        input_ids,
        token_type_ids,
        input_lengths,
        sequence_labels,
        token_labels,
        is_impossible_labels,
        choice_labels,
        input_mask,
    ):
        config.num_choices = self.num_choices
        model = XLMForMultipleChoice(config=config)
        model.to(torch_device)
        model.eval()
        multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
        multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
        multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
Sylvain Gugger's avatar
Sylvain Gugger committed
307
        result = model(
308
309
310
311
312
            multiple_choice_inputs_ids,
            attention_mask=multiple_choice_input_mask,
            token_type_ids=multiple_choice_token_type_ids,
            labels=choice_labels,
        )
Stas Bekman's avatar
Stas Bekman committed
313
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))
314

315
316
317
    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
318
319
320
321
322
323
324
            config,
            input_ids,
            token_type_ids,
            input_lengths,
            sequence_labels,
            token_labels,
            is_impossible_labels,
325
            choice_labels,
326
            input_mask,
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
        ) = config_and_inputs
        inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "lengths": input_lengths}
        return config, inputs_dict


@require_torch
class XLMModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (
        (
            XLMModel,
            XLMWithLMHeadModel,
            XLMForQuestionAnswering,
            XLMForSequenceClassification,
            XLMForQuestionAnsweringSimple,
342
            XLMForTokenClassification,
343
            XLMForMultipleChoice,
344
345
346
347
348
349
350
        )
        if is_torch_available()
        else ()
    )
    all_generative_model_classes = (
        (XLMWithLMHeadModel,) if is_torch_available() else ()
    )  # TODO (PVP): Check other models whether language generation is also applicable
thomwolf's avatar
thomwolf committed
351

thomwolf's avatar
thomwolf committed
352
    def setUp(self):
353
        self.model_tester = XLMModelTester(self)
thomwolf's avatar
thomwolf committed
354
        self.config_tester = ConfigTester(self, config_class=XLMConfig, emb_dim=37)
thomwolf's avatar
thomwolf committed
355
356

    def test_config(self):
thomwolf's avatar
thomwolf committed
357
        self.config_tester.run_common_tests()
thomwolf's avatar
thomwolf committed
358

thomwolf's avatar
thomwolf committed
359
360
361
    def test_xlm_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_model(*config_and_inputs)
thomwolf's avatar
thomwolf committed
362

thomwolf's avatar
thomwolf committed
363
364
365
    def test_xlm_lm_head(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_lm_head(*config_and_inputs)
thomwolf's avatar
thomwolf committed
366

367
368
369
370
    def test_xlm_simple_qa(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_simple_qa(*config_and_inputs)

thomwolf's avatar
thomwolf committed
371
372
373
    def test_xlm_qa(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_qa(*config_and_inputs)
thomwolf's avatar
thomwolf committed
374

thomwolf's avatar
thomwolf committed
375
376
377
    def test_xlm_sequence_classif(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_sequence_classif(*config_and_inputs)
thomwolf's avatar
thomwolf committed
378

379
    def test_xlm_token_classif(self):
380
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
381
        self.model_tester.create_and_check_xlm_token_classif(*config_and_inputs)
382

383
384
385
386
    def test_xlm_for_multiple_choice(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_for_multiple_choice(*config_and_inputs)

387
    @slow
thomwolf's avatar
thomwolf committed
388
    def test_model_from_pretrained(self):
389
        for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
390
            model = XLMModel.from_pretrained(model_name)
thomwolf's avatar
thomwolf committed
391
            self.assertIsNotNone(model)
392
393


394
@require_torch
395
396
397
398
class XLMModelLanguageGenerationTest(unittest.TestCase):
    @slow
    def test_lm_generate_xlm_mlm_en_2048(self):
        model = XLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048")
399
        model.to(torch_device)
400
        input_ids = torch.tensor([[14, 447]], dtype=torch.long, device=torch_device)  # the president
401
402
        expected_output_ids = [
            14,
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
        ]  # the president the president the president the president the president the president the president the president the president the president
        # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
        output_ids = model.generate(input_ids, do_sample=False)
425
        self.assertListEqual(output_ids[0].cpu().numpy().tolist(), expected_output_ids)