test_modeling_common.py 40.9 KB
Newer Older
thomwolf's avatar
thomwolf committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2019 HuggingFace Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Aymeric Augustin's avatar
Aymeric Augustin committed
15

16
import copy
Aymeric Augustin's avatar
Aymeric Augustin committed
17
import logging
18
import os.path
Aymeric Augustin's avatar
Aymeric Augustin committed
19
import random
20
import tempfile
thomwolf's avatar
thomwolf committed
21
import unittest
22
from typing import List
thomwolf's avatar
thomwolf committed
23

24
from transformers import is_torch_available
25

26
from .utils import require_multigpu, require_torch, slow, torch_device
27

Aymeric Augustin's avatar
Aymeric Augustin committed
28

29
if is_torch_available():
thomwolf's avatar
thomwolf committed
30
    import torch
31
    import numpy as np
thomwolf's avatar
thomwolf committed
32

33
34
35
36
37
38
    from transformers import (
        AdaptiveEmbedding,
        PretrainedConfig,
        PreTrainedModel,
        BertModel,
        BertConfig,
39
        BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
40
        MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
41
        MODEL_FOR_QUESTION_ANSWERING_MAPPING,
42
        top_k_top_p_filtering,
43
    )
thomwolf's avatar
thomwolf committed
44

45

46
47
48
def _config_zero_init(config):
    configs_no_init = copy.deepcopy(config)
    for key in configs_no_init.__dict__.keys():
49
        if "_range" in key or "_std" in key or "initializer_factor" in key:
Lysandre Debut's avatar
Lysandre Debut committed
50
            setattr(configs_no_init, key, 1e-10)
51
52
    return configs_no_init

thomwolf's avatar
thomwolf committed
53

54
55
56
57
58
@require_torch
class ModelTesterMixin:

    model_tester = None
    all_model_classes = ()
59
    all_generative_model_classes = ()
Patrick von Platen's avatar
Patrick von Platen committed
60
61
62
63
    test_torchscript = True
    test_pruning = True
    test_resize_embeddings = True
    test_head_masking = True
64
    test_missing_keys = True
65
66
    is_encoder_decoder = False

67
68
69
70
    def _prepare_for_class(self, inputs_dict, model_class):
        if model_class in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.values():
            return {
                k: v.unsqueeze(1).expand(-1, self.model_tester.num_choices, -1).contiguous()
Sylvain Gugger's avatar
Sylvain Gugger committed
71
72
                if isinstance(v, torch.Tensor) and v.ndim != 0
                else v
73
74
75
76
                for k, v in inputs_dict.items()
            }
        return inputs_dict

Patrick von Platen's avatar
Patrick von Platen committed
77
    def test_save_load(self):
78
79
80
81
82
83
84
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
85
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
86
            out_2 = outputs[0].cpu().numpy()
87
            out_2[np.isnan(out_2)] = 0
88

89
            with tempfile.TemporaryDirectory() as tmpdirname:
90
91
                model.save_pretrained(tmpdirname)
                model = model_class.from_pretrained(tmpdirname)
92
                model.to(torch_device)
93
                with torch.no_grad():
94
                    after_outputs = model(**self._prepare_for_class(inputs_dict, model_class))
thomwolf's avatar
thomwolf committed
95

96
97
98
                # Make sure we don't have nans
                out_1 = after_outputs[0].cpu().numpy()
                out_1[np.isnan(out_1)] = 0
thomwolf's avatar
thomwolf committed
99
100
                max_diff = np.amax(np.abs(out_1 - out_2))
                self.assertLessEqual(max_diff, 1e-5)
101

Patrick von Platen's avatar
Patrick von Platen committed
102
    def test_initialization(self):
103
104
105
106
107
108
109
110
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        configs_no_init = _config_zero_init(config)
        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            for name, param in model.named_parameters():
                if param.requires_grad:
                    self.assertIn(
Lysandre Debut's avatar
Lysandre Debut committed
111
                        ((param.data.mean() * 1e9).round() / 1e9).item(),
112
113
114
                        [0.0, 1.0],
                        msg="Parameter {} of model {} seems not properly initialized".format(name, model_class),
                    )
thomwolf's avatar
thomwolf committed
115

Patrick von Platen's avatar
Patrick von Platen committed
116
    def test_determinism(self):
117
118
119
120
121
122
123
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
124
125
                first = model(**self._prepare_for_class(inputs_dict, model_class))[0]
                second = model(**self._prepare_for_class(inputs_dict, model_class))[0]
126
127
128
129
130
131
132
            out_1 = first.cpu().numpy()
            out_2 = second.cpu().numpy()
            out_1 = out_1[~np.isnan(out_1)]
            out_2 = out_2[~np.isnan(out_2)]
            max_diff = np.amax(np.abs(out_1 - out_2))
            self.assertLessEqual(max_diff, 1e-5)

Patrick von Platen's avatar
Patrick von Platen committed
133
    def test_attention_outputs(self):
134
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
sshleifer's avatar
sshleifer committed
135
        seq_len = getattr(self.model_tester, "seq_length", None)
sshleifer's avatar
sshleifer committed
136
137
        decoder_seq_length = getattr(self.model_tester, "decoder_seq_length", seq_len)
        encoder_seq_length = getattr(self.model_tester, "encoder_seq_length", seq_len)
138
139
        decoder_key_length = getattr(self.model_tester, "key_length", decoder_seq_length)
        encoder_key_length = getattr(self.model_tester, "key_length", encoder_seq_length)
Patrick von Platen's avatar
Patrick von Platen committed
140
141
142
        chunk_length = getattr(self.model_tester, "chunk_length", None)
        if chunk_length is not None and hasattr(self.model_tester, "num_hashes"):
            encoder_seq_length = encoder_seq_length * self.model_tester.num_hashes
143
144

        for model_class in self.all_model_classes:
145
            inputs_dict["output_attentions"] = True
Joseph Liu's avatar
Joseph Liu committed
146
            inputs_dict["output_hidden_states"] = False
147
148
149
150
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
151
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
152
            attentions = outputs[-1]
153
154
155
156
157
158
159
160
161
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            # check that output_attentions also work using config
            del inputs_dict["output_attentions"]
            config.output_attentions = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
Sylvain Gugger's avatar
Sylvain Gugger committed
162
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
163
            attentions = outputs[-1]
164
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)
Patrick von Platen's avatar
Patrick von Platen committed
165
166
167
168
169
170
171
172
173
174
175

            if chunk_length is not None:
                self.assertListEqual(
                    list(attentions[0].shape[-4:]),
                    [self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length],
                )
            else:
                self.assertListEqual(
                    list(attentions[0].shape[-3:]),
                    [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
                )
176
            out_len = len(outputs)
thomwolf's avatar
thomwolf committed
177

178
            if self.is_encoder_decoder:
179
                correct_outlen = 4
Sam Shleifer's avatar
Sam Shleifer committed
180
                decoder_attention_idx = 1
181

182
183
184
185
186
187
188
                # loss is at first position
                if "labels" in inputs_dict:
                    correct_outlen += 1  # loss is added to beginning
                    decoder_attention_idx += 1
                # Question Answering model returns start_logits and end_logits
                if model_class in MODEL_FOR_QUESTION_ANSWERING_MAPPING.values():
                    correct_outlen += 1  # start_logits and end_logits instead of only 1 output
Sam Shleifer's avatar
Sam Shleifer committed
189
190
191
192
193
                    decoder_attention_idx += 1
                self.assertEqual(out_len, correct_outlen)

                decoder_attentions = outputs[decoder_attention_idx]
                self.assertIsInstance(decoder_attentions, (list, tuple))
194
                self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers)
thomwolf's avatar
thomwolf committed
195
                self.assertListEqual(
196
197
                    list(decoder_attentions[0].shape[-3:]),
                    [self.model_tester.num_attention_heads, decoder_seq_length, decoder_key_length],
198
                )
thomwolf's avatar
thomwolf committed
199

200
            # Check attention is always last and order is fine
201
            inputs_dict["output_attentions"] = True
Joseph Liu's avatar
Joseph Liu committed
202
            inputs_dict["output_hidden_states"] = True
203
204
205
206
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
207
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
208
209
210
211
            self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1), len(outputs))

            self_attentions = outputs[-1]
            self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers)
Patrick von Platen's avatar
Patrick von Platen committed
212
213
214
215
216
217
218
219
220
221
            if chunk_length is not None:
                self.assertListEqual(
                    list(self_attentions[0].shape[-4:]),
                    [self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length],
                )
            else:
                self.assertListEqual(
                    list(self_attentions[0].shape[-3:]),
                    [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
                )
thomwolf's avatar
thomwolf committed
222

Patrick von Platen's avatar
Patrick von Platen committed
223
    def test_torchscript(self):
224
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
thomwolf's avatar
thomwolf committed
225

226
        self._create_and_check_torchscript(config, inputs_dict)
thomwolf's avatar
thomwolf committed
227

Patrick von Platen's avatar
Patrick von Platen committed
228
    def test_torchscript_output_attentions(self):
229
230
231
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.output_attentions = True
        self._create_and_check_torchscript(config, inputs_dict)
thomwolf's avatar
thomwolf committed
232

Patrick von Platen's avatar
Patrick von Platen committed
233
    def test_torchscript_output_hidden_state(self):
234
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
235

236
237
        config.output_hidden_states = True
        self._create_and_check_torchscript(config, inputs_dict)
thomwolf's avatar
thomwolf committed
238

239
    def _create_and_check_torchscript(self, config, inputs_dict):
Patrick von Platen's avatar
Patrick von Platen committed
240
        if not self.test_torchscript:
241
            return
242

243
244
245
246
247
248
        configs_no_init = _config_zero_init(config)  # To be sure we have no Nan
        configs_no_init.torchscript = True
        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            model.to(torch_device)
            model.eval()
249
            inputs = self._prepare_for_class(inputs_dict, model_class)["input_ids"]  # Let's keep only input_ids
thomwolf's avatar
thomwolf committed
250

251
252
253
254
            try:
                traced_gpt2 = torch.jit.trace(model, inputs)
            except RuntimeError:
                self.fail("Couldn't trace module.")
thomwolf's avatar
thomwolf committed
255

256
            with tempfile.TemporaryDirectory() as tmp_dir_name:
257
                pt_file_name = os.path.join(tmp_dir_name, "traced_model.pt")
thomwolf's avatar
thomwolf committed
258

259
260
261
262
                try:
                    torch.jit.save(traced_gpt2, pt_file_name)
                except Exception:
                    self.fail("Couldn't save module.")
thomwolf's avatar
thomwolf committed
263

264
265
266
267
                try:
                    loaded_model = torch.jit.load(pt_file_name)
                except Exception:
                    self.fail("Couldn't load module.")
LysandreJik's avatar
LysandreJik committed
268

269
270
            model.to(torch_device)
            model.eval()
thomwolf's avatar
thomwolf committed
271

272
273
            loaded_model.to(torch_device)
            loaded_model.eval()
thomwolf's avatar
thomwolf committed
274

275
276
277
278
            model_state_dict = model.state_dict()
            loaded_model_state_dict = loaded_model.state_dict()

            self.assertEqual(set(model_state_dict.keys()), set(loaded_model_state_dict.keys()))
thomwolf's avatar
thomwolf committed
279

280
            models_equal = True
281
282
            for layer_name, p1 in model_state_dict.items():
                p2 = loaded_model_state_dict[layer_name]
283
284
                if p1.data.ne(p2.data).sum() > 0:
                    models_equal = False
thomwolf's avatar
thomwolf committed
285

286
            self.assertTrue(models_equal)
thomwolf's avatar
thomwolf committed
287

Patrick von Platen's avatar
Patrick von Platen committed
288
289
    def test_headmasking(self):
        if not self.test_head_masking:
290
            return
291

292
293
294
        global_rng.seed(42)
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        global_rng.seed()
LysandreJik's avatar
LysandreJik committed
295

296
        inputs_dict["output_attentions"] = True
297
298
299
300
301
302
        config.output_hidden_states = True
        configs_no_init = _config_zero_init(config)  # To be sure we have no Nan
        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            model.to(torch_device)
            model.eval()
LysandreJik's avatar
LysandreJik committed
303

304
305
306
            # Prepare head_mask
            # Set require_grad after having prepared the tensor to avoid error (leaf variable has been moved into the graph interior)
            head_mask = torch.ones(
307
                self.model_tester.num_hidden_layers, self.model_tester.num_attention_heads, device=torch_device,
308
309
310
311
            )
            head_mask[0, 0] = 0
            head_mask[-1, :-1] = 0
            head_mask.requires_grad_(requires_grad=True)
312
            inputs = self._prepare_for_class(inputs_dict, model_class).copy()
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
            inputs["head_mask"] = head_mask

            outputs = model(**inputs)

            # Test that we can get a gradient back for importance score computation
            output = sum(t.sum() for t in outputs[0])
            output = output.sum()
            output.backward()
            multihead_outputs = head_mask.grad

            attentions = outputs[-1]

            # Remove Nan
            for t in attentions:
                self.assertLess(
                    torch.sum(torch.isnan(t)), t.numel() / 4
                )  # Check we don't have more than 25% nans (arbitrary)
            attentions = [
                t.masked_fill(torch.isnan(t), 0.0) for t in attentions
            ]  # remove them (the test is less complete)

            self.assertIsNotNone(multihead_outputs)
            self.assertEqual(len(multihead_outputs), self.model_tester.num_hidden_layers)
            self.assertAlmostEqual(attentions[0][..., 0, :, :].flatten().sum().item(), 0.0)
            self.assertNotEqual(attentions[0][..., -1, :, :].flatten().sum().item(), 0.0)
            self.assertNotEqual(attentions[1][..., 0, :, :].flatten().sum().item(), 0.0)
            self.assertAlmostEqual(attentions[-1][..., -2, :, :].flatten().sum().item(), 0.0)
            self.assertNotEqual(attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0)

Patrick von Platen's avatar
Patrick von Platen committed
342
343
    def test_head_pruning(self):
        if not self.test_pruning:
344
345
346
            return

        for model_class in self.all_model_classes:
347
            (config, inputs_dict,) = self.model_tester.prepare_config_and_inputs_for_common()
348

349
350
            if "head_mask" in inputs_dict:
                del inputs_dict["head_mask"]
351

352
            inputs_dict["output_attentions"] = True
353
354
355
356
            config.output_hidden_states = False
            model = model_class(config=config)
            model.to(torch_device)
            model.eval()
357
358
359
360
            heads_to_prune = {
                0: list(range(1, self.model_tester.num_attention_heads)),
                -1: [0],
            }
361
362
            model.prune_heads(heads_to_prune)
            with torch.no_grad():
363
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
364

365
            attentions = outputs[-1]
366

367
368
369
            self.assertEqual(attentions[0].shape[-3], 1)
            self.assertEqual(attentions[1].shape[-3], self.model_tester.num_attention_heads)
            self.assertEqual(attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1)
LysandreJik's avatar
LysandreJik committed
370

Patrick von Platen's avatar
Patrick von Platen committed
371
372
    def test_head_pruning_save_load_from_pretrained(self):
        if not self.test_pruning:
373
            return
LysandreJik's avatar
LysandreJik committed
374

375
        for model_class in self.all_model_classes:
376
            (config, inputs_dict,) = self.model_tester.prepare_config_and_inputs_for_common()
377
378
379

            if "head_mask" in inputs_dict:
                del inputs_dict["head_mask"]
380

381
            inputs_dict["output_attentions"] = True
382
383
384
385
            config.output_hidden_states = False
            model = model_class(config=config)
            model.to(torch_device)
            model.eval()
386
387
388
389
            heads_to_prune = {
                0: list(range(1, self.model_tester.num_attention_heads)),
                -1: [0],
            }
390
            model.prune_heads(heads_to_prune)
391

392
            with tempfile.TemporaryDirectory() as temp_dir_name:
393
394
                model.save_pretrained(temp_dir_name)
                model = model_class.from_pretrained(temp_dir_name)
395
                model.to(torch_device)
396

397
            with torch.no_grad():
398
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
399
400
401
402
            attentions = outputs[-1]
            self.assertEqual(attentions[0].shape[-3], 1)
            self.assertEqual(attentions[1].shape[-3], self.model_tester.num_attention_heads)
            self.assertEqual(attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1)
403

Patrick von Platen's avatar
Patrick von Platen committed
404
405
    def test_head_pruning_save_load_from_config_init(self):
        if not self.test_pruning:
406
            return
407

408
        for model_class in self.all_model_classes:
409
            (config, inputs_dict,) = self.model_tester.prepare_config_and_inputs_for_common()
410

411
412
            if "head_mask" in inputs_dict:
                del inputs_dict["head_mask"]
413

414
            inputs_dict["output_attentions"] = True
415
            config.output_hidden_states = False
416

417
418
419
420
            heads_to_prune = {
                0: list(range(1, self.model_tester.num_attention_heads)),
                -1: [0],
            }
421
            config.pruned_heads = heads_to_prune
422

423
424
425
            model = model_class(config=config)
            model.to(torch_device)
            model.eval()
426

427
            with torch.no_grad():
428
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
429
            attentions = outputs[-1]
430

431
432
433
            self.assertEqual(attentions[0].shape[-3], 1)
            self.assertEqual(attentions[1].shape[-3], self.model_tester.num_attention_heads)
            self.assertEqual(attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1)
434

Patrick von Platen's avatar
Patrick von Platen committed
435
436
    def test_head_pruning_integration(self):
        if not self.test_pruning:
437
            return
438

439
        for model_class in self.all_model_classes:
440
            (config, inputs_dict,) = self.model_tester.prepare_config_and_inputs_for_common()
441

442
443
            if "head_mask" in inputs_dict:
                del inputs_dict["head_mask"]
444

445
            inputs_dict["output_attentions"] = True
446
            config.output_hidden_states = False
447

448
449
            heads_to_prune = {0: [0], 1: [1, 2]}
            config.pruned_heads = heads_to_prune
450

451
452
453
            model = model_class(config=config)
            model.to(torch_device)
            model.eval()
454

455
            with torch.no_grad():
456
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
457
            attentions = outputs[-1]
458

459
460
461
462
            self.assertEqual(attentions[0].shape[-3], self.model_tester.num_attention_heads - 1)
            self.assertEqual(attentions[1].shape[-3], self.model_tester.num_attention_heads - 2)
            self.assertEqual(attentions[2].shape[-3], self.model_tester.num_attention_heads)
            self.assertEqual(attentions[3].shape[-3], self.model_tester.num_attention_heads)
thomwolf's avatar
thomwolf committed
463

464
            with tempfile.TemporaryDirectory() as temp_dir_name:
465
466
                model.save_pretrained(temp_dir_name)
                model = model_class.from_pretrained(temp_dir_name)
467
                model.to(torch_device)
thomwolf's avatar
thomwolf committed
468

469
            with torch.no_grad():
470
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
471
            attentions = outputs[-1]
LysandreJik's avatar
LysandreJik committed
472

473
474
475
476
            self.assertEqual(attentions[0].shape[-3], self.model_tester.num_attention_heads - 1)
            self.assertEqual(attentions[1].shape[-3], self.model_tester.num_attention_heads - 2)
            self.assertEqual(attentions[2].shape[-3], self.model_tester.num_attention_heads)
            self.assertEqual(attentions[3].shape[-3], self.model_tester.num_attention_heads)
thomwolf's avatar
thomwolf committed
477

478
479
            heads_to_prune = {0: [0], 2: [1, 2]}
            model.prune_heads(heads_to_prune)
480

481
            with torch.no_grad():
482
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
483
            attentions = outputs[-1]
484

485
486
487
488
            self.assertEqual(attentions[0].shape[-3], self.model_tester.num_attention_heads - 1)
            self.assertEqual(attentions[1].shape[-3], self.model_tester.num_attention_heads - 2)
            self.assertEqual(attentions[2].shape[-3], self.model_tester.num_attention_heads - 2)
            self.assertEqual(attentions[3].shape[-3], self.model_tester.num_attention_heads)
489

490
            self.assertDictEqual(model.config.pruned_heads, {0: [0], 1: [1, 2], 2: [1, 2]})
thomwolf's avatar
thomwolf committed
491

Patrick von Platen's avatar
Patrick von Platen committed
492
    def test_hidden_states_output(self):
Joseph Liu's avatar
Joseph Liu committed
493
        def check_hidden_states_output(inputs_dict, config, model_class):
494
            model = model_class(config)
495
            model.to(torch_device)
thomwolf's avatar
thomwolf committed
496
            model.eval()
Joseph Liu's avatar
Joseph Liu committed
497

thomwolf's avatar
thomwolf committed
498
            with torch.no_grad():
499
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
500
            hidden_states = outputs[-1]
Patrick von Platen's avatar
Patrick von Platen committed
501

Joseph Liu's avatar
Joseph Liu committed
502
            self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1)
Patrick von Platen's avatar
Patrick von Platen committed
503
504
505
506
507
508
509
            if hasattr(self.model_tester, "encoder_seq_length"):
                seq_length = self.model_tester.encoder_seq_length
                if hasattr(self.model_tester, "chunk_length") and self.model_tester.chunk_length > 1:
                    seq_length = seq_length * self.model_tester.chunk_length
            else:
                seq_length = self.model_tester.seq_length

510
            self.assertListEqual(
Patrick von Platen's avatar
Patrick von Platen committed
511
                list(hidden_states[0].shape[-2:]), [seq_length, self.model_tester.hidden_size],
512
            )
thomwolf's avatar
thomwolf committed
513

Joseph Liu's avatar
Joseph Liu committed
514
515
516
517
518
519
520
521
522
523
524
525
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            inputs_dict["output_hidden_states"] = True
            check_hidden_states_output(inputs_dict, config, model_class)

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True

            check_hidden_states_output(inputs_dict, config, model_class)

Patrick von Platen's avatar
Patrick von Platen committed
526
    def test_resize_tokens_embeddings(self):
527
        (original_config, inputs_dict,) = self.model_tester.prepare_config_and_inputs_for_common()
Patrick von Platen's avatar
Patrick von Platen committed
528
        if not self.test_resize_embeddings:
529
530
531
532
533
            return

        for model_class in self.all_model_classes:
            config = copy.deepcopy(original_config)
            model = model_class(config)
534
            model.to(torch_device)
535

Patrick von Platen's avatar
Patrick von Platen committed
536
537
538
            if self.model_tester.is_training is False:
                model.eval()

539
540
541
542
543
544
545
546
547
548
            model_vocab_size = config.vocab_size
            # Retrieve the embeddings and clone theme
            model_embed = model.resize_token_embeddings(model_vocab_size)
            cloned_embeddings = model_embed.weight.clone()

            # Check that resizing the token embeddings with a larger vocab size increases the model's vocab size
            model_embed = model.resize_token_embeddings(model_vocab_size + 10)
            self.assertEqual(model.config.vocab_size, model_vocab_size + 10)
            # Check that it actually resizes the embeddings matrix
            self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] + 10)
549
            # Check that the model can still do a forward pass successfully (every parameter should be resized)
550
            model(**self._prepare_for_class(inputs_dict, model_class))
551
552
553
554
555
556
557

            # Check that resizing the token embeddings with a smaller vocab size decreases the model's vocab size
            model_embed = model.resize_token_embeddings(model_vocab_size - 15)
            self.assertEqual(model.config.vocab_size, model_vocab_size - 15)
            # Check that it actually resizes the embeddings matrix
            self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] - 15)

558
559
560
            # Check that the model can still do a forward pass successfully (every parameter should be resized)
            # Input ids should be clamped to the maximum size of the vocabulary
            inputs_dict["input_ids"].clamp_(max=model_vocab_size - 15 - 1)
561
            model(**self._prepare_for_class(inputs_dict, model_class))
562

563
564
565
566
567
568
569
570
            # Check that adding and removing tokens has not modified the first part of the embedding matrix.
            models_equal = True
            for p1, p2 in zip(cloned_embeddings, model_embed.weight):
                if p1.data.ne(p2.data).sum() > 0:
                    models_equal = False

            self.assertTrue(models_equal)

Patrick von Platen's avatar
Patrick von Platen committed
571
    def test_model_common_attributes(self):
572
573
574
575
576
577
578
579
580
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            self.assertIsInstance(model.get_input_embeddings(), (torch.nn.Embedding, AdaptiveEmbedding))
            model.set_input_embeddings(torch.nn.Embedding(10, 10))
            x = model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(x, torch.nn.Linear))

581
    def test_correct_missing_keys(self):
582
583
        if not self.test_missing_keys:
            return
584
585
586
587
588
589
590
591
592
593
594
595
596
597
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            base_model_prefix = model.base_model_prefix

            if hasattr(model, base_model_prefix):
                with tempfile.TemporaryDirectory() as temp_dir_name:
                    model.base_model.save_pretrained(temp_dir_name)
                    model, loading_info = model_class.from_pretrained(temp_dir_name, output_loading_info=True)

                    with self.subTest(msg="Missing keys for {}".format(model.__class__.__name__)):
                        self.assertGreater(len(loading_info["missing_keys"]), 0)

598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
    def test_tie_model_weights(self):
        if not self.test_torchscript:
            return

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        def check_same_values(layer_1, layer_2):
            equal = True
            for p1, p2 in zip(layer_1.weight, layer_2.weight):
                if p1.data.ne(p2.data).sum() > 0:
                    equal = False
            return equal

        for model_class in self.all_model_classes:
            config.torchscript = True
            model_not_tied = model_class(config)
            if model_not_tied.get_output_embeddings() is None:
                continue

            params_not_tied = list(model_not_tied.parameters())

            config_tied = copy.deepcopy(config)
            config_tied.torchscript = False
            model_tied = model_class(config_tied)
            params_tied = list(model_tied.parameters())

            # Check that the embedding layer and decoding layer are the same in size and in value
            self.assertGreater(len(params_not_tied), len(params_tied))
            # self.assertTrue(check_same_values(embeddings, decoding))

            # # Check that after modification, they remain the same.
            # embeddings.weight.data.div_(2)
            # # Check that the embedding layer and decoding layer are the same in size and in value
            # self.assertTrue(embeddings.weight.shape, decoding.weight.shape)
            # self.assertTrue(check_same_values(embeddings, decoding))

            # # Check that after modification, they remain the same.
            # decoding.weight.data.div_(4)
            # # Check that the embedding layer and decoding layer are the same in size and in value
            # self.assertTrue(embeddings.weight.shape, decoding.weight.shape)
            # self.assertTrue(check_same_values(embeddings, decoding))

            # Check that after resize they remain tied.
            model_tied.resize_token_embeddings(config.vocab_size + 10)
            params_tied_2 = list(model_tied.parameters())
            self.assertGreater(len(params_not_tied), len(params_tied))
            self.assertEqual(len(params_tied_2), len(params_tied))

            # decoding.weight.data.mul_(20)
            # # Check that the embedding layer and decoding layer are the same in size and in value
            # self.assertTrue(model.transformer.wte.weight.shape, model.lm_head.weight.shape)
            # self.assertTrue(check_same_values(model.transformer.wte, model.lm_head))

Patrick von Platen's avatar
Patrick von Platen committed
651
    def test_inputs_embeds(self):
Sam Shleifer's avatar
Sam Shleifer committed
652

653
654
655
656
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
657
            model.to(torch_device)
thomwolf's avatar
thomwolf committed
658
            model.eval()
659

660
661
662
663
664
665
666
667
668
669
            inputs = copy.deepcopy(self._prepare_for_class(inputs_dict, model_class))
            if not self.is_encoder_decoder:
                input_ids = inputs["input_ids"]
                del inputs["input_ids"]
            else:
                encoder_input_ids = inputs["input_ids"]
                decoder_input_ids = inputs.get("decoder_input_ids", encoder_input_ids)
                del inputs["input_ids"]
                inputs.pop("decoder_input_ids", None)

670
671
            wte = model.get_input_embeddings()
            if not self.is_encoder_decoder:
672
                inputs["inputs_embeds"] = wte(input_ids)
673
            else:
674
675
                inputs["inputs_embeds"] = wte(encoder_input_ids)
                inputs["decoder_inputs_embeds"] = wte(decoder_input_ids)
676

thomwolf's avatar
thomwolf committed
677
            with torch.no_grad():
678
                model(**inputs)
679

680
    def test_lm_head_model_random_no_beam_search_generate(self):
681
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
682
        input_ids = inputs_dict["input_ids"] if "input_ids" in inputs_dict else inputs_dict["inputs"]
683

Patrick von Platen's avatar
Patrick von Platen committed
684
685
686
        # make sure that input_ids is at most of size 15
        input_ids = input_ids[..., :15]

687
        # iterate over all generative models
688
        for model_class in self.all_generative_model_classes:
689
            model = model_class(config).to(torch_device)
Patrick von Platen's avatar
Patrick von Platen committed
690
            model.eval()
691
692

            if config.bos_token_id is None:
693
                # if bos token id is not defined, model needs input_ids
694
                with self.assertRaises(AssertionError):
695
                    model.generate(do_sample=True, max_length=5)
696
                # num_return_sequences = 1
697
                self._check_generated_ids(model.generate(input_ids, do_sample=True))
698
            else:
699
                # num_return_sequences = 1
700
                self._check_generated_ids(model.generate(do_sample=True, max_length=5))
701

702
            with self.assertRaises(AssertionError):
703
                # generating multiple sequences when no beam search generation
704
705
706
                # is not allowed as it would always generate the same sequences
                model.generate(input_ids, do_sample=False, num_return_sequences=2)

707
708
            # num_return_sequences > 1, sample
            self._check_generated_ids(model.generate(input_ids, do_sample=True, num_return_sequences=2))
709
710

            # check bad words tokens language generation
711
            # create list of 1-seq bad token and list of 2-seq of bad tokens
712
713
714
715
            bad_words_ids = [
                self._generate_random_bad_tokens(1, model.config),
                self._generate_random_bad_tokens(2, model.config),
            ]
716
            output_tokens = model.generate(
717
                input_ids, do_sample=True, bad_words_ids=bad_words_ids, num_return_sequences=2
718
            )
719
            # only count generated tokens
720
721
            generated_ids = output_tokens[:, input_ids.shape[-1] :]
            self.assertFalse(self._check_match_tokens(generated_ids.tolist(), bad_words_ids))
722

723
724
    def test_lm_head_model_random_beam_search_generate(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
725
726
727
        input_ids = (inputs_dict["input_ids"] if "input_ids" in inputs_dict else inputs_dict["inputs"]).to(
            torch_device
        )
728

Patrick von Platen's avatar
Patrick von Platen committed
729
730
731
        # make sure that input_ids is at most of size 15
        input_ids = input_ids[..., :15]

732
        for model_class in self.all_generative_model_classes:
733
            model = model_class(config).to(torch_device)
Patrick von Platen's avatar
Patrick von Platen committed
734
            model.eval()
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753

            if config.bos_token_id is None:
                # if bos token id is not defined mobel needs input_ids, num_return_sequences = 1
                self._check_generated_ids(model.generate(input_ids, do_sample=True, num_beams=2))
            else:
                # num_return_sequences = 1
                self._check_generated_ids(model.generate(do_sample=True, max_length=5, num_beams=2))

            with self.assertRaises(AssertionError):
                # generating more sequences than having beams leads is not possible
                model.generate(input_ids, do_sample=False, num_return_sequences=3, num_beams=2)

            # num_return_sequences > 1, sample
            self._check_generated_ids(model.generate(input_ids, do_sample=True, num_beams=2, num_return_sequences=2,))
            # num_return_sequences > 1, greedy
            self._check_generated_ids(model.generate(input_ids, do_sample=False, num_beams=2, num_return_sequences=2))

            # check bad words tokens language generation
            # create list of 1-seq bad token and list of 2-seq of bad tokens
754
755
756
757
            bad_words_ids = [
                self._generate_random_bad_tokens(1, model.config),
                self._generate_random_bad_tokens(2, model.config),
            ]
758
            output_tokens = model.generate(
759
                input_ids, do_sample=False, bad_words_ids=bad_words_ids, num_beams=2, num_return_sequences=2
760
            )
761
            # only count generated tokens
762
763
764
            generated_ids = output_tokens[:, input_ids.shape[-1] :]
            self.assertFalse(self._check_match_tokens(generated_ids.tolist(), bad_words_ids))

765
    def _generate_random_bad_tokens(self, num_bad_tokens: int, config) -> List[int]:
766
        # special tokens cannot be bad tokens
767
        special_tokens = [x for x in [config.bos_token_id, config.eos_token_id, config.pad_token_id] if x is not None]
768
769
770
        # create random bad tokens that are not special tokens
        bad_tokens = []
        while len(bad_tokens) < num_bad_tokens:
771
            token = ids_tensor((1, 1), self.model_tester.vocab_size).squeeze(0).cpu().numpy()[0]
772
773
774
775
            if token not in special_tokens:
                bad_tokens.append(token)
        return bad_tokens

776
    def _check_generated_ids(self, output_ids):
777
778
779
780
        for token_id in output_ids[0].tolist():
            self.assertGreaterEqual(token_id, 0)
            self.assertLess(token_id, self.model_tester.vocab_size)

781
782
783
784
785
786
787
788
789
790
791
792
    def _check_match_tokens(self, generated_ids, bad_words_ids):
        # for all bad word tokens
        for bad_word_ids in bad_words_ids:
            # for all slices in batch
            for generated_ids_slice in generated_ids:
                # for all word idx
                for i in range(len(bad_word_ids), len(generated_ids_slice)):
                    # if tokens match
                    if generated_ids_slice[i - len(bad_word_ids) : i] == bad_word_ids:
                        return True
        return False

793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
    @require_multigpu
    def test_multigpu_data_parallel_forward(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        # some params shouldn't be scattered by nn.DataParallel
        # so just remove them if they are present.
        blacklist_non_batched_params = ["head_mask"]
        for k in blacklist_non_batched_params:
            inputs_dict.pop(k, None)

        # move input tensors to cuda:O
        for k, v in inputs_dict.items():
            if torch.is_tensor(v):
                inputs_dict[k] = v.to(0)

        for model_class in self.all_model_classes:
            model = model_class(config=config)
            model.to(0)
            model.eval()

            # Wrap model in nn.DataParallel
            model = torch.nn.DataParallel(model)
            with torch.no_grad():
                _ = model(**inputs_dict)

818

819
global_rng = random.Random()
thomwolf's avatar
thomwolf committed
820
821


thomwolf's avatar
thomwolf committed
822
def ids_tensor(shape, vocab_size, rng=None, name=None):
823
    #  Creates a random int32 tensor of the shape within the vocab size
thomwolf's avatar
thomwolf committed
824
    if rng is None:
825
        rng = global_rng
thomwolf's avatar
thomwolf committed
826

thomwolf's avatar
thomwolf committed
827
828
829
    total_dims = 1
    for dim in shape:
        total_dims *= dim
thomwolf's avatar
thomwolf committed
830

thomwolf's avatar
thomwolf committed
831
832
833
    values = []
    for _ in range(total_dims):
        values.append(rng.randint(0, vocab_size - 1))
thomwolf's avatar
thomwolf committed
834

835
    return torch.tensor(data=values, dtype=torch.long, device=torch_device).view(shape).contiguous()
thomwolf's avatar
thomwolf committed
836
837


838
def floats_tensor(shape, scale=1.0, rng=None, name=None):
Patrick von Platen's avatar
Patrick von Platen committed
839
    """Creates a random float32 tensor"""
840
841
842
843
844
845
846
847
848
849
850
    if rng is None:
        rng = global_rng

    total_dims = 1
    for dim in shape:
        total_dims *= dim

    values = []
    for _ in range(total_dims):
        values.append(rng.random() * scale)

851
    return torch.tensor(data=values, dtype=torch.float, device=torch_device).view(shape).contiguous()
852
853


854
@require_torch
thomwolf's avatar
thomwolf committed
855
class ModelUtilsTest(unittest.TestCase):
856
    @slow
Patrick von Platen's avatar
Patrick von Platen committed
857
    def test_model_from_pretrained(self):
thomwolf's avatar
thomwolf committed
858
        logging.basicConfig(level=logging.INFO)
859
        for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
thomwolf's avatar
thomwolf committed
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
            config = BertConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, PretrainedConfig)

            model = BertModel.from_pretrained(model_name)
            model, loading_info = BertModel.from_pretrained(model_name, output_loading_info=True)
            self.assertIsNotNone(model)
            self.assertIsInstance(model, PreTrainedModel)
            for value in loading_info.values():
                self.assertEqual(len(value), 0)

            config = BertConfig.from_pretrained(model_name, output_attentions=True, output_hidden_states=True)
            model = BertModel.from_pretrained(model_name, output_attentions=True, output_hidden_states=True)
            self.assertEqual(model.config.output_hidden_states, True)
            self.assertEqual(model.config, config)
875
876
877
878
879
880


@require_torch
class UtilsFunctionsTest(unittest.TestCase):

    # tests whether the top_k_top_p function behaves as expected
Patrick von Platen's avatar
Patrick von Platen committed
881
    def test_top_k_top_p_filtering(self):
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
        logits = torch.tensor(
            [
                [
                    8.2220991,  # 3rd highest value; idx. 0
                    -0.5620044,
                    5.23229752,
                    4.0386393,
                    -6.8798378,
                    -0.54785802,
                    -3.2012153,
                    2.92777176,
                    1.88171953,
                    7.35341276,  # 5th highest value; idx. 9
                    8.43207833,  # 2nd highest value; idx. 10
                    -9.85711836,
                    -5.96209236,
                    -1.13039161,
                    -7.1115294,
                    -0.8369633,
                    -5.3186408,
                    7.06427407,
                    0.81369344,
                    -0.82023817,
                    -5.9179796,
                    0.58813443,
                    -6.99778438,
                    4.71551189,
                    -0.18771637,
                    7.44020759,  # 4th highest value; idx. 25
                    9.38450987,  # 1st highest value; idx. 26
                    2.12662941,
                    -9.32562038,
                    2.35652522,
                ],  # cummulative prob of 5 highest values <= 0.6
                [
                    0.58425518,
                    4.53139238,
                    -5.57510464,
                    -6.28030699,
                    -7.19529503,
                    -4.02122551,
                    1.39337037,
                    -6.06707057,
                    1.59480517,
                    -9.643119,
                    0.03907799,
                    0.67231762,
                    -8.88206726,
                    6.27115922,  # 4th highest value; idx. 13
                    2.28520723,
                    4.82767506,
                    4.30421368,
                    8.8275313,  # 2nd highest value; idx. 17
                    5.44029958,  # 5th highest value; idx. 18
                    -4.4735794,
                    7.38579536,  # 3rd highest value; idx. 20
                    -2.91051663,
                    2.61946077,
                    -2.5674762,
                    -9.48959302,
                    -4.02922645,
                    -1.35416918,
                    9.67702323,  # 1st highest value; idx. 27
                    -5.89478553,
                    1.85370467,
                ],  # cummulative prob of 5 highest values <= 0.6
            ],
            dtype=torch.float,
            device=torch_device,
        )

        non_inf_expected_idx = torch.tensor(
            [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]],
            dtype=torch.long,
            device=torch_device,
        )  # expected non filtered idx as noted above

        non_inf_expected_output = torch.tensor(
            [
                8.2221,
                7.3534,
                8.4321,
                7.4402,
                9.3845,
                6.2712,
                8.8275,
                5.4403,
                7.3858,
                9.6770,
            ],  # expected non filtered values as noted above
            dtype=torch.float,
            device=torch_device,
        )

        output = top_k_top_p_filtering(logits, top_k=10, top_p=0.6, min_tokens_to_keep=4)
        non_inf_output = output[output != -float("inf")].to(device=torch_device)
        non_inf_idx = (output != -float("inf")).nonzero().to(device=torch_device)

        self.assertTrue(torch.allclose(non_inf_expected_output, non_inf_output, atol=1e-12))
        self.assertTrue(torch.all(torch.eq(non_inf_expected_idx, non_inf_idx)))