test_modeling_auto.py 17.1 KB
Newer Older
thomwolf's avatar
thomwolf committed
1
# coding=utf-8
Sylvain Gugger's avatar
Sylvain Gugger committed
2
# Copyright 2020 The HuggingFace Team. All rights reserved.
thomwolf's avatar
thomwolf committed
3
4
5
6
7
8
9
10
11
12
13
14
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Aymeric Augustin's avatar
Aymeric Augustin committed
15

16
import copy
17
import sys
18
import tempfile
Aymeric Augustin's avatar
Aymeric Augustin committed
19
import unittest
20
from pathlib import Path
thomwolf's avatar
thomwolf committed
21

22
23
from transformers import BertConfig, is_torch_available
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
24
from transformers.testing_utils import (
25
    DUMMY_UNKNOWN_IDENTIFIER,
26
27
28
29
30
    SMALL_MODEL_IDENTIFIER,
    require_scatter,
    require_torch,
    slow,
)
Aymeric Augustin's avatar
Aymeric Augustin committed
31

32
from ..bert.test_modeling_bert import BertModelTester
33

34

35
sys.path.append(str(Path(__file__).parent.parent.parent / "utils"))
36
37
38
39

from test_module.custom_configuration import CustomConfig  # noqa E402


40
if is_torch_available():
41
42
    import torch

43
    from test_module.custom_modeling import CustomModel
44
45
46
    from transformers import (
        AutoConfig,
        AutoModel,
47
48
        AutoModelForCausalLM,
        AutoModelForMaskedLM,
49
50
        AutoModelForPreTraining,
        AutoModelForQuestionAnswering,
51
        AutoModelForSeq2SeqLM,
52
        AutoModelForSequenceClassification,
53
        AutoModelForTableQuestionAnswering,
54
        AutoModelForTokenClassification,
55
56
57
58
59
        AutoModelWithLMHead,
        BertForMaskedLM,
        BertForPreTraining,
        BertForQuestionAnswering,
        BertForSequenceClassification,
60
        BertForTokenClassification,
61
        BertModel,
62
63
        FunnelBaseModel,
        FunnelModel,
64
65
66
67
68
        GPT2Config,
        GPT2LMHeadModel,
        RobertaForMaskedLM,
        T5Config,
        T5ForConditionalGeneration,
69
70
        TapasConfig,
        TapasForQuestionAnswering,
71
    )
Sylvain Gugger's avatar
Sylvain Gugger committed
72
    from transformers.models.auto.modeling_auto import (
73
74
        MODEL_FOR_CAUSAL_LM_MAPPING,
        MODEL_FOR_MASKED_LM_MAPPING,
Lysandre's avatar
Lysandre committed
75
76
        MODEL_FOR_PRETRAINING_MAPPING,
        MODEL_FOR_QUESTION_ANSWERING_MAPPING,
77
        MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
Lysandre's avatar
Lysandre committed
78
        MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
79
        MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
Lysandre's avatar
Lysandre committed
80
        MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
81
        MODEL_MAPPING,
Lysandre's avatar
Lysandre committed
82
83
        MODEL_WITH_LM_HEAD_MAPPING,
    )
Sylvain Gugger's avatar
Sylvain Gugger committed
84
85
86
    from transformers.models.bert.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_LIST
    from transformers.models.gpt2.modeling_gpt2 import GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
    from transformers.models.t5.modeling_t5 import T5_PRETRAINED_MODEL_ARCHIVE_LIST
87
    from transformers.models.tapas.modeling_tapas import TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST
thomwolf's avatar
thomwolf committed
88
89


90
@require_torch
thomwolf's avatar
thomwolf committed
91
class AutoModelTest(unittest.TestCase):
92
    @slow
thomwolf's avatar
thomwolf committed
93
    def test_model_from_pretrained(self):
94
        for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
thomwolf's avatar
thomwolf committed
95
96
97
98
99
100
101
102
            config = AutoConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, BertConfig)

            model = AutoModel.from_pretrained(model_name)
            model, loading_info = AutoModel.from_pretrained(model_name, output_loading_info=True)
            self.assertIsNotNone(model)
            self.assertIsInstance(model, BertModel)
Lysandre Debut's avatar
Lysandre Debut committed
103
104
105
106
107

            self.assertEqual(len(loading_info["missing_keys"]), 0)
            self.assertEqual(len(loading_info["unexpected_keys"]), 8)
            self.assertEqual(len(loading_info["mismatched_keys"]), 0)
            self.assertEqual(len(loading_info["error_msgs"]), 0)
thomwolf's avatar
thomwolf committed
108

thomwolf's avatar
thomwolf committed
109
110
    @slow
    def test_model_for_pretraining_from_pretrained(self):
111
        for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
thomwolf's avatar
thomwolf committed
112
113
114
115
116
117
118
119
            config = AutoConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, BertConfig)

            model = AutoModelForPreTraining.from_pretrained(model_name)
            model, loading_info = AutoModelForPreTraining.from_pretrained(model_name, output_loading_info=True)
            self.assertIsNotNone(model)
            self.assertIsInstance(model, BertForPreTraining)
120
121
122
            # Only one value should not be initialized and in the missing keys.
            missing_keys = loading_info.pop("missing_keys")
            self.assertListEqual(["cls.predictions.decoder.bias"], missing_keys)
123
            for key, value in loading_info.items():
124
                self.assertEqual(len(value), 0)
thomwolf's avatar
thomwolf committed
125

126
    @slow
LysandreJik's avatar
LysandreJik committed
127
    def test_lmhead_model_from_pretrained(self):
128
        for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
LysandreJik's avatar
LysandreJik committed
129
130
131
132
133
134
135
136
137
            config = AutoConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, BertConfig)

            model = AutoModelWithLMHead.from_pretrained(model_name)
            model, loading_info = AutoModelWithLMHead.from_pretrained(model_name, output_loading_info=True)
            self.assertIsNotNone(model)
            self.assertIsInstance(model, BertForMaskedLM)

138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
    @slow
    def test_model_for_causal_lm(self):
        for model_name in GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            config = AutoConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, GPT2Config)

            model = AutoModelForCausalLM.from_pretrained(model_name)
            model, loading_info = AutoModelForCausalLM.from_pretrained(model_name, output_loading_info=True)
            self.assertIsNotNone(model)
            self.assertIsInstance(model, GPT2LMHeadModel)

    @slow
    def test_model_for_masked_lm(self):
        for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            config = AutoConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, BertConfig)

            model = AutoModelForMaskedLM.from_pretrained(model_name)
            model, loading_info = AutoModelForMaskedLM.from_pretrained(model_name, output_loading_info=True)
            self.assertIsNotNone(model)
            self.assertIsInstance(model, BertForMaskedLM)

    @slow
    def test_model_for_encoder_decoder_lm(self):
        for model_name in T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            config = AutoConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, T5Config)

            model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
            model, loading_info = AutoModelForSeq2SeqLM.from_pretrained(model_name, output_loading_info=True)
            self.assertIsNotNone(model)
            self.assertIsInstance(model, T5ForConditionalGeneration)

174
    @slow
LysandreJik's avatar
LysandreJik committed
175
    def test_sequence_classification_model_from_pretrained(self):
176
        for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
LysandreJik's avatar
LysandreJik committed
177
178
179
180
181
            config = AutoConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, BertConfig)

            model = AutoModelForSequenceClassification.from_pretrained(model_name)
182
183
184
            model, loading_info = AutoModelForSequenceClassification.from_pretrained(
                model_name, output_loading_info=True
            )
LysandreJik's avatar
LysandreJik committed
185
186
187
            self.assertIsNotNone(model)
            self.assertIsInstance(model, BertForSequenceClassification)

188
    @slow
LysandreJik's avatar
LysandreJik committed
189
    def test_question_answering_model_from_pretrained(self):
190
        for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
LysandreJik's avatar
LysandreJik committed
191
192
193
194
195
196
197
198
199
            config = AutoConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, BertConfig)

            model = AutoModelForQuestionAnswering.from_pretrained(model_name)
            model, loading_info = AutoModelForQuestionAnswering.from_pretrained(model_name, output_loading_info=True)
            self.assertIsNotNone(model)
            self.assertIsInstance(model, BertForQuestionAnswering)

200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
    @slow
    @require_scatter
    def test_table_question_answering_model_from_pretrained(self):
        for model_name in TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
            config = AutoConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, TapasConfig)

            model = AutoModelForTableQuestionAnswering.from_pretrained(model_name)
            model, loading_info = AutoModelForTableQuestionAnswering.from_pretrained(
                model_name, output_loading_info=True
            )
            self.assertIsNotNone(model)
            self.assertIsInstance(model, TapasForQuestionAnswering)

215
216
    @slow
    def test_token_classification_model_from_pretrained(self):
217
        for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
218
219
220
221
222
223
224
225
226
            config = AutoConfig.from_pretrained(model_name)
            self.assertIsNotNone(config)
            self.assertIsInstance(config, BertConfig)

            model = AutoModelForTokenClassification.from_pretrained(model_name)
            model, loading_info = AutoModelForTokenClassification.from_pretrained(model_name, output_loading_info=True)
            self.assertIsNotNone(model)
            self.assertIsInstance(model, BertForTokenClassification)

Julien Chaumond's avatar
Julien Chaumond committed
227
228
229
    def test_from_pretrained_identifier(self):
        model = AutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER)
        self.assertIsInstance(model, BertForMaskedLM)
230
231
        self.assertEqual(model.num_parameters(), 14410)
        self.assertEqual(model.num_parameters(only_trainable=True), 14410)
Julien Chaumond's avatar
Julien Chaumond committed
232
233

    def test_from_identifier_from_model_type(self):
234
        model = AutoModelWithLMHead.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER)
Julien Chaumond's avatar
Julien Chaumond committed
235
        self.assertIsInstance(model, RobertaForMaskedLM)
236
237
        self.assertEqual(model.num_parameters(), 14410)
        self.assertEqual(model.num_parameters(only_trainable=True), 14410)
Lysandre's avatar
Lysandre committed
238

239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
    def test_from_pretrained_with_tuple_values(self):
        # For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel
        model = AutoModel.from_pretrained("sgugger/funnel-random-tiny")
        self.assertIsInstance(model, FunnelModel)

        config = copy.deepcopy(model.config)
        config.architectures = ["FunnelBaseModel"]
        model = AutoModel.from_config(config)
        self.assertIsInstance(model, FunnelBaseModel)

        with tempfile.TemporaryDirectory() as tmp_dir:
            model.save_pretrained(tmp_dir)
            model = AutoModel.from_pretrained(tmp_dir)
            self.assertIsInstance(model, FunnelBaseModel)

Lysandre's avatar
Lysandre committed
254
255
256
257
258
259
260
261
    def test_parents_and_children_in_mappings(self):
        # Test that the children are placed before the parents in the mappings, as the `instanceof` will be triggered
        # by the parents and will return the wrong configuration type when using auto models

        mappings = (
            MODEL_MAPPING,
            MODEL_FOR_PRETRAINING_MAPPING,
            MODEL_FOR_QUESTION_ANSWERING_MAPPING,
262
            MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
Lysandre's avatar
Lysandre committed
263
264
265
            MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
            MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
            MODEL_WITH_LM_HEAD_MAPPING,
266
267
268
            MODEL_FOR_CAUSAL_LM_MAPPING,
            MODEL_FOR_MASKED_LM_MAPPING,
            MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
Lysandre's avatar
Lysandre committed
269
270
271
272
273
274
        )

        for mapping in mappings:
            mapping = tuple(mapping.items())
            for index, (child_config, child_model) in enumerate(mapping[1:]):
                for parent_config, parent_model in mapping[: index + 1]:
Sam Shleifer's avatar
Sam Shleifer committed
275
276
                    assert not issubclass(
                        child_config, parent_config
Lysandre Debut's avatar
Lysandre Debut committed
277
                    ), f"{child_config.__name__} is child of {parent_config.__name__}"
278
279
280
281
282
283
284
285
286

                    # Tuplify child_model and parent_model since some of them could be tuples.
                    if not isinstance(child_model, (list, tuple)):
                        child_model = (child_model,)
                    if not isinstance(parent_model, (list, tuple)):
                        parent_model = (parent_model,)

                    for child, parent in [(a, b) for a in child_model for b in parent_model]:
                        assert not issubclass(child, parent), f"{child.__name__} is child of {parent.__name__}"
287

288
    def test_from_pretrained_dynamic_model_local(self):
289
290
291
        try:
            AutoConfig.register("custom", CustomConfig)
            AutoModel.register(CustomConfig, CustomModel)
292

293
294
            config = CustomConfig(hidden_size=32)
            model = CustomModel(config)
295

296
297
298
299
300
301
302
303
304
305
306
307
            with tempfile.TemporaryDirectory() as tmp_dir:
                model.save_pretrained(tmp_dir)

                new_model = AutoModel.from_pretrained(tmp_dir, trust_remote_code=True)
                for p1, p2 in zip(model.parameters(), new_model.parameters()):
                    self.assertTrue(torch.equal(p1, p2))

        finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in MODEL_MAPPING._extra_content:
                del MODEL_MAPPING._extra_content[CustomConfig]
308

309
310
311
312
313
314
315
316
    def test_from_pretrained_dynamic_model_distant(self):
        model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model", trust_remote_code=True)
        self.assertEqual(model.__class__.__name__, "NewModel")

        # This one uses a relative import to a util file, this checks it is downloaded and used properly.
        model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model_with_util", trust_remote_code=True)
        self.assertEqual(model.__class__.__name__, "NewModel")

317
    def test_new_model_registration(self):
318
        AutoConfig.register("custom", CustomConfig)
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334

        auto_classes = [
            AutoModel,
            AutoModelForCausalLM,
            AutoModelForMaskedLM,
            AutoModelForPreTraining,
            AutoModelForQuestionAnswering,
            AutoModelForSequenceClassification,
            AutoModelForTokenClassification,
        ]

        try:
            for auto_class in auto_classes:
                with self.subTest(auto_class.__name__):
                    # Wrong config class will raise an error
                    with self.assertRaises(ValueError):
335
336
                        auto_class.register(BertConfig, CustomModel)
                    auto_class.register(CustomConfig, CustomModel)
337
338
339
340
341
342
                    # Trying to register something existing in the Transformers library will raise an error
                    with self.assertRaises(ValueError):
                        auto_class.register(BertConfig, BertModel)

                    # Now that the config is registered, it can be used as any other config with the auto-API
                    tiny_config = BertModelTester(self).get_config()
343
                    config = CustomConfig(**tiny_config.to_dict())
344
                    model = auto_class.from_config(config)
345
                    self.assertIsInstance(model, CustomModel)
346
347
348
349

                    with tempfile.TemporaryDirectory() as tmp_dir:
                        model.save_pretrained(tmp_dir)
                        new_model = auto_class.from_pretrained(tmp_dir)
350
351
                        # The model is a CustomModel but from the new dynamically imported class.
                        self.assertIsInstance(new_model, CustomModel)
352
353

        finally:
354
355
            if "custom" in CONFIG_MAPPING._extra_content:
                del CONFIG_MAPPING._extra_content["custom"]
356
357
358
359
360
361
362
363
364
            for mapping in (
                MODEL_MAPPING,
                MODEL_FOR_PRETRAINING_MAPPING,
                MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                MODEL_FOR_CAUSAL_LM_MAPPING,
                MODEL_FOR_MASKED_LM_MAPPING,
            ):
365
366
                if CustomConfig in mapping._extra_content:
                    del mapping._extra_content[CustomConfig]
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393

    def test_repo_not_found(self):
        with self.assertRaisesRegex(
            EnvironmentError, "bert-base is not a local folder and is not a valid model identifier"
        ):
            _ = AutoModel.from_pretrained("bert-base")

    def test_revision_not_found(self):
        with self.assertRaisesRegex(
            EnvironmentError, r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"
        ):
            _ = AutoModel.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER, revision="aaaaaa")

    def test_model_file_not_found(self):
        with self.assertRaisesRegex(
            EnvironmentError,
            "hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin",
        ):
            _ = AutoModel.from_pretrained("hf-internal-testing/config-no-model")

    def test_model_from_tf_suggestion(self):
        with self.assertRaisesRegex(EnvironmentError, "Use `from_tf=True` to load this model"):
            _ = AutoModel.from_pretrained("hf-internal-testing/tiny-bert-tf-only")

    def test_model_from_flax_suggestion(self):
        with self.assertRaisesRegex(EnvironmentError, "Use `from_flax=True` to load this model"):
            _ = AutoModel.from_pretrained("hf-internal-testing/tiny-bert-flax-only")