modeling_auto.py 64.1 KB
Newer Older
thomwolf's avatar
thomwolf committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# coding=utf-8
# Copyright 2018 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Auto Model class. """


18
import warnings
Julien Chaumond's avatar
Julien Chaumond committed
19
from collections import OrderedDict
thomwolf's avatar
thomwolf committed
20

21
22
from .configuration_auto import (
    AlbertConfig,
23
    AutoConfig,
Sam Shleifer's avatar
Sam Shleifer committed
24
    BartConfig,
25
    BertConfig,
26
    BertGenerationConfig,
27
28
    CamembertConfig,
    CTRLConfig,
Pengcheng He's avatar
Pengcheng He committed
29
    DebertaConfig,
30
    DistilBertConfig,
Ola Piktus's avatar
Ola Piktus committed
31
    DPRConfig,
Lysandre Debut's avatar
Lysandre Debut committed
32
    ElectraConfig,
33
    EncoderDecoderConfig,
Lysandre's avatar
Lysandre committed
34
    FlaubertConfig,
35
    FSMTConfig,
Sylvain Gugger's avatar
Sylvain Gugger committed
36
    FunnelConfig,
37
    GPT2Config,
Minghao Li's avatar
Minghao Li committed
38
    LayoutLMConfig,
Iz Beltagy's avatar
Iz Beltagy committed
39
    LongformerConfig,
40
    LxmertConfig,
41
    MBartConfig,
Vasily Shamporov's avatar
Vasily Shamporov committed
42
    MobileBertConfig,
43
    OpenAIGPTConfig,
44
    PegasusConfig,
Patrick von Platen's avatar
Patrick von Platen committed
45
    ReformerConfig,
Yacine Jernite's avatar
Yacine Jernite committed
46
    RetriBertConfig,
47
    RobertaConfig,
48
    T5Config,
49
50
51
    TransfoXLConfig,
    XLMConfig,
    XLMRobertaConfig,
Aymeric Augustin's avatar
Aymeric Augustin committed
52
    XLNetConfig,
53
    replace_list_option_in_docstrings,
Aymeric Augustin's avatar
Aymeric Augustin committed
54
)
55
from .configuration_marian import MarianConfig
56
from .configuration_utils import PretrainedConfig
57
from .file_utils import add_start_docstrings
Aymeric Augustin's avatar
Aymeric Augustin committed
58
59
from .modeling_albert import (
    AlbertForMaskedLM,
60
    AlbertForMultipleChoice,
61
    AlbertForPreTraining,
Aymeric Augustin's avatar
Aymeric Augustin committed
62
63
    AlbertForQuestionAnswering,
    AlbertForSequenceClassification,
64
    AlbertForTokenClassification,
Aymeric Augustin's avatar
Aymeric Augustin committed
65
    AlbertModel,
66
)
Suraj Patil's avatar
Suraj Patil committed
67
68
69
70
71
72
from .modeling_bart import (
    BartForConditionalGeneration,
    BartForQuestionAnswering,
    BartForSequenceClassification,
    BartModel,
)
73
74
from .modeling_bert import (
    BertForMaskedLM,
Julien Chaumond's avatar
Julien Chaumond committed
75
    BertForMultipleChoice,
thomwolf's avatar
thomwolf committed
76
    BertForPreTraining,
77
    BertForQuestionAnswering,
Aymeric Augustin's avatar
Aymeric Augustin committed
78
    BertForSequenceClassification,
79
    BertForTokenClassification,
80
    BertLMHeadModel,
Aymeric Augustin's avatar
Aymeric Augustin committed
81
    BertModel,
82
)
83
from .modeling_bert_generation import BertGenerationDecoder, BertGenerationEncoder
Aymeric Augustin's avatar
Aymeric Augustin committed
84
from .modeling_camembert import (
Suraj Patil's avatar
Suraj Patil committed
85
    CamembertForCausalLM,
Aymeric Augustin's avatar
Aymeric Augustin committed
86
    CamembertForMaskedLM,
Julien Chaumond's avatar
Julien Chaumond committed
87
    CamembertForMultipleChoice,
88
    CamembertForQuestionAnswering,
Aymeric Augustin's avatar
Aymeric Augustin committed
89
90
91
    CamembertForSequenceClassification,
    CamembertForTokenClassification,
    CamembertModel,
92
)
93
from .modeling_ctrl import CTRLLMHeadModel, CTRLModel
Pengcheng He's avatar
Pengcheng He committed
94
from .modeling_deberta import DebertaForSequenceClassification, DebertaModel
95
96
from .modeling_distilbert import (
    DistilBertForMaskedLM,
97
    DistilBertForMultipleChoice,
Aymeric Augustin's avatar
Aymeric Augustin committed
98
    DistilBertForQuestionAnswering,
99
100
    DistilBertForSequenceClassification,
    DistilBertForTokenClassification,
Aymeric Augustin's avatar
Aymeric Augustin committed
101
    DistilBertModel,
102
)
Ola Piktus's avatar
Ola Piktus committed
103
from .modeling_dpr import DPRQuestionEncoder
Lysandre Debut's avatar
Lysandre Debut committed
104
105
from .modeling_electra import (
    ElectraForMaskedLM,
Suraj Patil's avatar
Suraj Patil committed
106
    ElectraForMultipleChoice,
Lysandre Debut's avatar
Lysandre Debut committed
107
    ElectraForPreTraining,
108
    ElectraForQuestionAnswering,
109
    ElectraForSequenceClassification,
Lysandre Debut's avatar
Lysandre Debut committed
110
111
112
    ElectraForTokenClassification,
    ElectraModel,
)
113
from .modeling_encoder_decoder import EncoderDecoderModel
Lysandre's avatar
Lysandre committed
114
from .modeling_flaubert import (
115
    FlaubertForMultipleChoice,
116
    FlaubertForQuestionAnsweringSimple,
Lysandre's avatar
Lysandre committed
117
    FlaubertForSequenceClassification,
118
    FlaubertForTokenClassification,
Lysandre's avatar
Lysandre committed
119
120
121
    FlaubertModel,
    FlaubertWithLMHeadModel,
)
122
from .modeling_fsmt import FSMTForConditionalGeneration, FSMTModel
Sylvain Gugger's avatar
Sylvain Gugger committed
123
124
125
126
127
128
129
130
from .modeling_funnel import (
    FunnelForMaskedLM,
    FunnelForMultipleChoice,
    FunnelForQuestionAnswering,
    FunnelForSequenceClassification,
    FunnelForTokenClassification,
    FunnelModel,
)
131
from .modeling_gpt2 import GPT2LMHeadModel, GPT2Model
Minghao Li's avatar
Minghao Li committed
132
from .modeling_layoutlm import LayoutLMForMaskedLM, LayoutLMForTokenClassification, LayoutLMModel
133
134
from .modeling_longformer import (
    LongformerForMaskedLM,
135
    LongformerForMultipleChoice,
136
    LongformerForQuestionAnswering,
137
    LongformerForSequenceClassification,
138
    LongformerForTokenClassification,
139
140
    LongformerModel,
)
141
from .modeling_lxmert import LxmertForPreTraining, LxmertModel
142
from .modeling_marian import MarianMTModel
143
from .modeling_mbart import MBartForConditionalGeneration
Vasily Shamporov's avatar
Vasily Shamporov committed
144
145
146
147
148
149
150
151
152
from .modeling_mobilebert import (
    MobileBertForMaskedLM,
    MobileBertForMultipleChoice,
    MobileBertForPreTraining,
    MobileBertForQuestionAnswering,
    MobileBertForSequenceClassification,
    MobileBertForTokenClassification,
    MobileBertModel,
)
153
from .modeling_openai import OpenAIGPTLMHeadModel, OpenAIGPTModel
154
from .modeling_pegasus import PegasusForConditionalGeneration
Ola Piktus's avatar
Ola Piktus committed
155
156
157
158
159
from .modeling_rag import (  # noqa: F401 - need to import all RagModels to be in globals() function
    RagModel,
    RagSequenceForGeneration,
    RagTokenForGeneration,
)
160
161
162
163
164
165
from .modeling_reformer import (
    ReformerForMaskedLM,
    ReformerForQuestionAnswering,
    ReformerModel,
    ReformerModelWithLMHead,
)
Yacine Jernite's avatar
Yacine Jernite committed
166
from .modeling_retribert import RetriBertModel
Aymeric Augustin's avatar
Aymeric Augustin committed
167
from .modeling_roberta import (
168
    RobertaForCausalLM,
Aymeric Augustin's avatar
Aymeric Augustin committed
169
    RobertaForMaskedLM,
Julien Chaumond's avatar
Julien Chaumond committed
170
    RobertaForMultipleChoice,
Julien Chaumond's avatar
Julien Chaumond committed
171
    RobertaForQuestionAnswering,
Aymeric Augustin's avatar
Aymeric Augustin committed
172
173
174
    RobertaForSequenceClassification,
    RobertaForTokenClassification,
    RobertaModel,
175
)
176
177
from .modeling_t5 import T5ForConditionalGeneration, T5Model
from .modeling_transfo_xl import TransfoXLLMHeadModel, TransfoXLModel
Aymeric Augustin's avatar
Aymeric Augustin committed
178
from .modeling_xlm import (
179
    XLMForMultipleChoice,
180
    XLMForQuestionAnsweringSimple,
Aymeric Augustin's avatar
Aymeric Augustin committed
181
    XLMForSequenceClassification,
182
    XLMForTokenClassification,
Aymeric Augustin's avatar
Aymeric Augustin committed
183
184
    XLMModel,
    XLMWithLMHeadModel,
185
186
)
from .modeling_xlm_roberta import (
187
    XLMRobertaForCausalLM,
188
    XLMRobertaForMaskedLM,
Julien Chaumond's avatar
Julien Chaumond committed
189
    XLMRobertaForMultipleChoice,
190
    XLMRobertaForQuestionAnswering,
Aymeric Augustin's avatar
Aymeric Augustin committed
191
    XLMRobertaForSequenceClassification,
192
    XLMRobertaForTokenClassification,
Aymeric Augustin's avatar
Aymeric Augustin committed
193
194
195
    XLMRobertaModel,
)
from .modeling_xlnet import (
Julien Chaumond's avatar
Julien Chaumond committed
196
    XLNetForMultipleChoice,
197
    XLNetForQuestionAnsweringSimple,
Aymeric Augustin's avatar
Aymeric Augustin committed
198
199
200
201
    XLNetForSequenceClassification,
    XLNetForTokenClassification,
    XLNetLMHeadModel,
    XLNetModel,
202
)
Lysandre Debut's avatar
Lysandre Debut committed
203
from .utils import logging
thomwolf's avatar
thomwolf committed
204

thomwolf's avatar
thomwolf committed
205

Lysandre Debut's avatar
Lysandre Debut committed
206
logger = logging.get_logger(__name__)
thomwolf's avatar
thomwolf committed
207
208


Julien Chaumond's avatar
Julien Chaumond committed
209
MODEL_MAPPING = OrderedDict(
Julien Chaumond's avatar
Julien Chaumond committed
210
    [
Yacine Jernite's avatar
Yacine Jernite committed
211
        (RetriBertConfig, RetriBertModel),
Julien Chaumond's avatar
Julien Chaumond committed
212
213
214
215
        (T5Config, T5Model),
        (DistilBertConfig, DistilBertModel),
        (AlbertConfig, AlbertModel),
        (CamembertConfig, CamembertModel),
216
        (XLMRobertaConfig, XLMRobertaModel),
Sam Shleifer's avatar
Sam Shleifer committed
217
        (BartConfig, BartModel),
Iz Beltagy's avatar
Iz Beltagy committed
218
        (LongformerConfig, LongformerModel),
219
        (RobertaConfig, RobertaModel),
Minghao Li's avatar
Minghao Li committed
220
        (LayoutLMConfig, LayoutLMModel),
Julien Chaumond's avatar
Julien Chaumond committed
221
222
223
        (BertConfig, BertModel),
        (OpenAIGPTConfig, OpenAIGPTModel),
        (GPT2Config, GPT2Model),
Vasily Shamporov's avatar
Vasily Shamporov committed
224
        (MobileBertConfig, MobileBertModel),
Julien Chaumond's avatar
Julien Chaumond committed
225
226
        (TransfoXLConfig, TransfoXLModel),
        (XLNetConfig, XLNetModel),
Lysandre's avatar
Lysandre committed
227
        (FlaubertConfig, FlaubertModel),
228
        (FSMTConfig, FSMTModel),
Julien Chaumond's avatar
Julien Chaumond committed
229
230
        (XLMConfig, XLMModel),
        (CTRLConfig, CTRLModel),
Lysandre Debut's avatar
Lysandre Debut committed
231
        (ElectraConfig, ElectraModel),
Patrick von Platen's avatar
Patrick von Platen committed
232
        (ReformerConfig, ReformerModel),
Sylvain Gugger's avatar
Sylvain Gugger committed
233
        (FunnelConfig, FunnelModel),
234
        (LxmertConfig, LxmertModel),
235
        (BertGenerationConfig, BertGenerationEncoder),
Pengcheng He's avatar
Pengcheng He committed
236
        (DebertaConfig, DebertaModel),
Ola Piktus's avatar
Ola Piktus committed
237
        (DPRConfig, DPRQuestionEncoder),
Julien Chaumond's avatar
Julien Chaumond committed
238
239
240
    ]
)

thomwolf's avatar
thomwolf committed
241
242
MODEL_FOR_PRETRAINING_MAPPING = OrderedDict(
    [
Minghao Li's avatar
Minghao Li committed
243
        (LayoutLMConfig, LayoutLMForMaskedLM),
Yacine Jernite's avatar
Yacine Jernite committed
244
        (RetriBertConfig, RetriBertModel),
245
        (T5Config, T5ForConditionalGeneration),
thomwolf's avatar
thomwolf committed
246
        (DistilBertConfig, DistilBertForMaskedLM),
247
        (AlbertConfig, AlbertForPreTraining),
thomwolf's avatar
thomwolf committed
248
249
        (CamembertConfig, CamembertForMaskedLM),
        (XLMRobertaConfig, XLMRobertaForMaskedLM),
250
        (BartConfig, BartForConditionalGeneration),
251
        (FSMTConfig, FSMTForConditionalGeneration),
Iz Beltagy's avatar
Iz Beltagy committed
252
        (LongformerConfig, LongformerForMaskedLM),
thomwolf's avatar
thomwolf committed
253
254
255
256
        (RobertaConfig, RobertaForMaskedLM),
        (BertConfig, BertForPreTraining),
        (OpenAIGPTConfig, OpenAIGPTLMHeadModel),
        (GPT2Config, GPT2LMHeadModel),
Vasily Shamporov's avatar
Vasily Shamporov committed
257
        (MobileBertConfig, MobileBertForPreTraining),
thomwolf's avatar
thomwolf committed
258
259
        (TransfoXLConfig, TransfoXLLMHeadModel),
        (XLNetConfig, XLNetLMHeadModel),
Lysandre's avatar
Lysandre committed
260
        (FlaubertConfig, FlaubertWithLMHeadModel),
thomwolf's avatar
thomwolf committed
261
262
        (XLMConfig, XLMWithLMHeadModel),
        (CTRLConfig, CTRLLMHeadModel),
Lysandre Debut's avatar
Lysandre Debut committed
263
        (ElectraConfig, ElectraForPreTraining),
264
        (LxmertConfig, LxmertForPreTraining),
thomwolf's avatar
thomwolf committed
265
266
267
    ]
)

Julien Chaumond's avatar
Julien Chaumond committed
268
MODEL_WITH_LM_HEAD_MAPPING = OrderedDict(
269
    [
Minghao Li's avatar
Minghao Li committed
270
        (LayoutLMConfig, LayoutLMForMaskedLM),
271
        (T5Config, T5ForConditionalGeneration),
272
273
274
275
        (DistilBertConfig, DistilBertForMaskedLM),
        (AlbertConfig, AlbertForMaskedLM),
        (CamembertConfig, CamembertForMaskedLM),
        (XLMRobertaConfig, XLMRobertaForMaskedLM),
276
        (MarianConfig, MarianMTModel),
277
        (FSMTConfig, FSMTForConditionalGeneration),
278
        (BartConfig, BartForConditionalGeneration),
Iz Beltagy's avatar
Iz Beltagy committed
279
        (LongformerConfig, LongformerForMaskedLM),
280
        (RobertaConfig, RobertaForMaskedLM),
281
282
283
        (BertConfig, BertForMaskedLM),
        (OpenAIGPTConfig, OpenAIGPTLMHeadModel),
        (GPT2Config, GPT2LMHeadModel),
Vasily Shamporov's avatar
Vasily Shamporov committed
284
        (MobileBertConfig, MobileBertForMaskedLM),
285
286
        (TransfoXLConfig, TransfoXLLMHeadModel),
        (XLNetConfig, XLNetLMHeadModel),
Lysandre's avatar
Lysandre committed
287
        (FlaubertConfig, FlaubertWithLMHeadModel),
288
289
        (XLMConfig, XLMWithLMHeadModel),
        (CTRLConfig, CTRLLMHeadModel),
Lysandre Debut's avatar
Lysandre Debut committed
290
        (ElectraConfig, ElectraForMaskedLM),
291
        (EncoderDecoderConfig, EncoderDecoderModel),
Patrick von Platen's avatar
Patrick von Platen committed
292
        (ReformerConfig, ReformerModelWithLMHead),
Sylvain Gugger's avatar
Sylvain Gugger committed
293
        (FunnelConfig, FunnelForMaskedLM),
294
295
296
    ]
)

297
298
MODEL_FOR_CAUSAL_LM_MAPPING = OrderedDict(
    [
Suraj Patil's avatar
Suraj Patil committed
299
        (CamembertConfig, CamembertForCausalLM),
300
        (XLMRobertaConfig, XLMRobertaForCausalLM),
301
        (RobertaConfig, RobertaForCausalLM),
302
303
304
305
306
307
308
309
310
311
312
        (BertConfig, BertLMHeadModel),
        (OpenAIGPTConfig, OpenAIGPTLMHeadModel),
        (GPT2Config, GPT2LMHeadModel),
        (TransfoXLConfig, TransfoXLLMHeadModel),
        (XLNetConfig, XLNetLMHeadModel),
        (
            XLMConfig,
            XLMWithLMHeadModel,
        ),  # XLM can be MLM and CLM => model should be split similar to BERT; leave here for now
        (CTRLConfig, CTRLLMHeadModel),
        (ReformerConfig, ReformerModelWithLMHead),
313
        (BertGenerationConfig, BertGenerationDecoder),
314
315
316
317
318
    ]
)

MODEL_FOR_MASKED_LM_MAPPING = OrderedDict(
    [
Minghao Li's avatar
Minghao Li committed
319
        (LayoutLMConfig, LayoutLMForMaskedLM),
320
321
        (DistilBertConfig, DistilBertForMaskedLM),
        (AlbertConfig, AlbertForMaskedLM),
322
        (BartConfig, BartForConditionalGeneration),
323
324
325
326
327
        (CamembertConfig, CamembertForMaskedLM),
        (XLMRobertaConfig, XLMRobertaForMaskedLM),
        (LongformerConfig, LongformerForMaskedLM),
        (RobertaConfig, RobertaForMaskedLM),
        (BertConfig, BertForMaskedLM),
Vasily Shamporov's avatar
Vasily Shamporov committed
328
        (MobileBertConfig, MobileBertForMaskedLM),
329
330
331
        (FlaubertConfig, FlaubertWithLMHeadModel),
        (XLMConfig, XLMWithLMHeadModel),
        (ElectraConfig, ElectraForMaskedLM),
332
        (ReformerConfig, ReformerForMaskedLM),
Sylvain Gugger's avatar
Sylvain Gugger committed
333
        (FunnelConfig, FunnelForMaskedLM),
334
335
336
337
338
339
    ]
)

MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
    [
        (T5Config, T5ForConditionalGeneration),
340
        (PegasusConfig, PegasusForConditionalGeneration),
341
        (MarianConfig, MarianMTModel),
342
        (MBartConfig, MBartForConditionalGeneration),
343
        (BartConfig, BartForConditionalGeneration),
344
        (FSMTConfig, FSMTForConditionalGeneration),
345
346
347
348
        (EncoderDecoderConfig, EncoderDecoderModel),
    ]
)

Julien Chaumond's avatar
Julien Chaumond committed
349
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
350
351
352
353
354
    [
        (DistilBertConfig, DistilBertForSequenceClassification),
        (AlbertConfig, AlbertForSequenceClassification),
        (CamembertConfig, CamembertForSequenceClassification),
        (XLMRobertaConfig, XLMRobertaForSequenceClassification),
Sam Shleifer's avatar
Sam Shleifer committed
355
        (BartConfig, BartForSequenceClassification),
356
        (LongformerConfig, LongformerForSequenceClassification),
357
        (RobertaConfig, RobertaForSequenceClassification),
358
359
        (BertConfig, BertForSequenceClassification),
        (XLNetConfig, XLNetForSequenceClassification),
Vasily Shamporov's avatar
Vasily Shamporov committed
360
        (MobileBertConfig, MobileBertForSequenceClassification),
Lysandre's avatar
Lysandre committed
361
        (FlaubertConfig, FlaubertForSequenceClassification),
Lysandre's avatar
Lysandre committed
362
        (XLMConfig, XLMForSequenceClassification),
363
        (ElectraConfig, ElectraForSequenceClassification),
Sylvain Gugger's avatar
Sylvain Gugger committed
364
        (FunnelConfig, FunnelForSequenceClassification),
Pengcheng He's avatar
Pengcheng He committed
365
        (DebertaConfig, DebertaForSequenceClassification),
366
367
368
    ]
)

Julien Chaumond's avatar
Julien Chaumond committed
369
MODEL_FOR_QUESTION_ANSWERING_MAPPING = OrderedDict(
370
371
372
    [
        (DistilBertConfig, DistilBertForQuestionAnswering),
        (AlbertConfig, AlbertForQuestionAnswering),
373
        (CamembertConfig, CamembertForQuestionAnswering),
Suraj Patil's avatar
Suraj Patil committed
374
        (BartConfig, BartForQuestionAnswering),
375
        (LongformerConfig, LongformerForQuestionAnswering),
376
        (XLMRobertaConfig, XLMRobertaForQuestionAnswering),
Malte Pietsch's avatar
Malte Pietsch committed
377
        (RobertaConfig, RobertaForQuestionAnswering),
378
        (BertConfig, BertForQuestionAnswering),
379
380
        (XLNetConfig, XLNetForQuestionAnsweringSimple),
        (FlaubertConfig, FlaubertForQuestionAnsweringSimple),
Vasily Shamporov's avatar
Vasily Shamporov committed
381
        (MobileBertConfig, MobileBertForQuestionAnswering),
382
        (XLMConfig, XLMForQuestionAnsweringSimple),
383
        (ElectraConfig, ElectraForQuestionAnswering),
384
        (ReformerConfig, ReformerForQuestionAnswering),
Sylvain Gugger's avatar
Sylvain Gugger committed
385
        (FunnelConfig, FunnelForQuestionAnswering),
386
387
388
    ]
)

Julien Chaumond's avatar
Julien Chaumond committed
389
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING = OrderedDict(
Julien Chaumond's avatar
Julien Chaumond committed
390
    [
Minghao Li's avatar
Minghao Li committed
391
        (LayoutLMConfig, LayoutLMForTokenClassification),
Julien Chaumond's avatar
Julien Chaumond committed
392
393
        (DistilBertConfig, DistilBertForTokenClassification),
        (CamembertConfig, CamembertForTokenClassification),
394
        (FlaubertConfig, FlaubertForTokenClassification),
395
        (XLMConfig, XLMForTokenClassification),
396
        (XLMRobertaConfig, XLMRobertaForTokenClassification),
397
        (LongformerConfig, LongformerForTokenClassification),
398
        (RobertaConfig, RobertaForTokenClassification),
Julien Chaumond's avatar
Julien Chaumond committed
399
        (BertConfig, BertForTokenClassification),
Vasily Shamporov's avatar
Vasily Shamporov committed
400
        (MobileBertConfig, MobileBertForTokenClassification),
Julien Chaumond's avatar
Julien Chaumond committed
401
        (XLNetConfig, XLNetForTokenClassification),
402
        (AlbertConfig, AlbertForTokenClassification),
Lysandre Debut's avatar
Lysandre Debut committed
403
        (ElectraConfig, ElectraForTokenClassification),
404
        (FlaubertConfig, FlaubertForTokenClassification),
Sylvain Gugger's avatar
Sylvain Gugger committed
405
        (FunnelConfig, FunnelForTokenClassification),
Julien Chaumond's avatar
Julien Chaumond committed
406
407
408
    ]
)

Julien Chaumond's avatar
Julien Chaumond committed
409
410
411
MODEL_FOR_MULTIPLE_CHOICE_MAPPING = OrderedDict(
    [
        (CamembertConfig, CamembertForMultipleChoice),
Suraj Patil's avatar
Suraj Patil committed
412
        (ElectraConfig, ElectraForMultipleChoice),
Julien Chaumond's avatar
Julien Chaumond committed
413
        (XLMRobertaConfig, XLMRobertaForMultipleChoice),
414
        (LongformerConfig, LongformerForMultipleChoice),
Julien Chaumond's avatar
Julien Chaumond committed
415
416
        (RobertaConfig, RobertaForMultipleChoice),
        (BertConfig, BertForMultipleChoice),
417
        (DistilBertConfig, DistilBertForMultipleChoice),
Vasily Shamporov's avatar
Vasily Shamporov committed
418
        (MobileBertConfig, MobileBertForMultipleChoice),
Julien Chaumond's avatar
Julien Chaumond committed
419
        (XLNetConfig, XLNetForMultipleChoice),
420
        (AlbertConfig, AlbertForMultipleChoice),
421
422
        (XLMConfig, XLMForMultipleChoice),
        (FlaubertConfig, FlaubertForMultipleChoice),
Sylvain Gugger's avatar
Sylvain Gugger committed
423
        (FunnelConfig, FunnelForMultipleChoice),
Julien Chaumond's avatar
Julien Chaumond committed
424
425
426
    ]
)

427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
AUTO_MODEL_PRETRAINED_DOCSTRING = r"""

        The model class to instantiate is selected based on the :obj:`model_type` property of the config object
        (either passed as an argument or loaded from :obj:`pretrained_model_name_or_path` if possible), or when it's
        missing, by falling back to using pattern matching on :obj:`pretrained_model_name_or_path`:

        List options

        The model is set in evaluation mode by default using ``model.eval()`` (so for instance, dropout modules are
        deactivated). To train the model, you should first set it back in training mode with ``model.train()``

        Args:
            pretrained_model_name_or_path:
                Can be either:

                    - A string with the `shortcut name` of a pretrained model to load from cache or download, e.g.,
                      ``bert-base-uncased``.
                    - A string with the `identifier name` of a pretrained model that was user-uploaded to our S3, e.g.,
                      ``dbmdz/bert-base-german-cased``.
                    - A path to a `directory` containing model weights saved using
                      :func:`~transformers.PreTrainedModel.save_pretrained`, e.g., ``./my_model_directory/``.
                    - A path or url to a `tensorflow index checkpoint file` (e.g, ``./tf_model/model.ckpt.index``). In
                      this case, ``from_tf`` should be set to :obj:`True` and a configuration object should be provided
                      as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in
                      a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
            model_args (additional positional arguments, `optional`):
                Will be passed along to the underlying model ``__init__()`` method.
            config (:class:`~transformers.PretrainedConfig`, `optional`):
                Configuration for the model to use instead of an automatically loaded configuation. Configuration can
                be automatically loaded when:

                    - The model is a model provided by the library (loaded with the `shortcut name` string of a
                      pretrained model).
                    - The model was saved using :meth:`~transformers.PreTrainedModel.save_pretrained` and is reloaded
                      by suppling the save directory.
                    - The model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a
                      configuration JSON file named `config.json` is found in the directory.
            state_dict (`Dict[str, torch.Tensor]`, `optional`):
                A state dictionary to use instead of a state dictionary loaded from saved weights file.

                This option can be used if you want to create a model from a pretrained configuration but load your own
                weights. In this case though, you should check if using
                :func:`~transformers.PreTrainedModel.save_pretrained` and
                :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option.
            cache_dir (:obj:`str`, `optional`):
                Path to a directory in which a downloaded pretrained model configuration should be cached if the
                standard cache should not be used.
            from_tf (:obj:`bool`, `optional`, defaults to :obj:`False`):
                Load the model weights from a TensorFlow checkpoint save file (see docstring of
                ``pretrained_model_name_or_path`` argument).
            force_download (:obj:`bool`, `optional`, defaults to :obj:`False`):
                Whether or not to force the (re-)download of the model weights and configuration files, overriding the
                cached versions if they exist.
            resume_download (:obj:`bool`, `optional`, defaults to :obj:`False`):
                Whether or not to delete incompletely received files. Will attempt to resume the download if such a
                file exists.
            proxies (:obj:`Dict[str, str], `optional`):
                A dictionary of proxy servers to use by protocol or endpoint, e.g.,
                :obj:`{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each
                request.
            output_loading_info(:obj:`bool`, `optional`, defaults to :obj:`False`):
                Whether ot not to also return a dictionnary containing missing keys, unexpected keys and error
                messages.
            local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`):
                Whether or not to only look at local files (e.g., not try doanloading the model).
            use_cdn(:obj:`bool`, `optional`, defaults to :obj:`True`):
                Whether or not to use Cloudfront (a Content Delivery Network, or CDN) when searching for the model on
                our S3 (faster). Should be set to :obj:`False` for checkpoints larger than 20GB.
            kwargs (additional keyword arguments, `optional`):
                Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
                :obj:`output_attentions=True`). Behaves differently depending on whether a ``config`` is provided or
                automatically loaded:

                    - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the
                      underlying model's ``__init__`` method (we assume all relevant updates to the configuration have
                      already been done)
                    - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class
                      initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of
                      ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute
                      with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration
                      attribute will be passed to the underlying model's ``__init__`` function.
"""


Julien Chaumond's avatar
Julien Chaumond committed
511
class AutoModel:
thomwolf's avatar
thomwolf committed
512
    r"""
513
514
515
    This is a generic model class that will be instantiated as one of the base model classes of the library
    when created with the when created with the :meth:`~transformers.AutoModel.from_pretrained` class method or the
    :meth:`~transformers.AutoModel.from_config` class methods.
thomwolf's avatar
thomwolf committed
516

517
    This class cannot be instantiated directly using ``__init__()`` (throws an error).
thomwolf's avatar
thomwolf committed
518
    """
519

thomwolf's avatar
thomwolf committed
520
    def __init__(self):
521
522
        raise EnvironmentError(
            "AutoModel is designed to be instantiated "
523
            "using the `AutoModel.from_pretrained(pretrained_model_name_or_path)` or "
524
525
            "`AutoModel.from_config(config)` methods."
        )
526
527

    @classmethod
528
    @replace_list_option_in_docstrings(MODEL_MAPPING, use_model_types=False)
529
    def from_config(cls, config):
530
531
        r"""
        Instantiates one of the base model classes of the library from a configuration.
532

Lysandre's avatar
Lysandre committed
533
534
        Note:
            Loading a model from its configuration file does **not** load the model weights.
535
536
            It only affects the model's configuration. Use :meth:`~transformers.AutoModel.from_pretrained` to load
            the model weights.
Lysandre's avatar
Lysandre committed
537

Lysandre's avatar
Lysandre committed
538
539
        Args:
            config (:class:`~transformers.PretrainedConfig`):
540
                The model class to instantiate is selected based on the configuration class:
Lysandre's avatar
Lysandre committed
541

542
                List options
543
544
545

        Examples::

546
547
548
549
            >>> from transformers import AutoConfig, AutoModel
            >>> # Download configuration from S3 and cache.
            >>> config = AutoConfig.from_pretrained('bert-base-uncased')
            >>> model = AutoModel.from_config(config)
550
        """
551
552
        if type(config) in MODEL_MAPPING.keys():
            return MODEL_MAPPING[type(config)](config)
553
554
555
556
557
558
        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_MAPPING.keys())
            )
        )
thomwolf's avatar
thomwolf committed
559
560

    @classmethod
561
    @replace_list_option_in_docstrings(MODEL_MAPPING)
562
563
564
565
    @add_start_docstrings(
        "Instantiate one of the base model classes of the library from a pretrained model.",
        AUTO_MODEL_PRETRAINED_DOCSTRING,
    )
thomwolf's avatar
thomwolf committed
566
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
567
        r"""
thomwolf's avatar
thomwolf committed
568

569
        Examples::
thomwolf's avatar
thomwolf committed
570

571
            >>> from transformers import AutoConfig, AutoModel
thomwolf's avatar
thomwolf committed
572

573
574
            >>> # Download model and configuration from S3 and cache.
            >>> model = AutoModel.from_pretrained('bert-base-uncased')
thomwolf's avatar
thomwolf committed
575

576
577
578
579
            >>> # Update configuration during loading
            >>> model = AutoModel.from_pretrained('bert-base-uncased', output_attentions=True)
            >>> model.config.output_attentions
            True
thomwolf's avatar
thomwolf committed
580

581
582
583
            >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            >>> config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
            >>> model = AutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
thomwolf's avatar
thomwolf committed
584
        """
585
586
        config = kwargs.pop("config", None)
        if not isinstance(config, PretrainedConfig):
587
588
589
            config, kwargs = AutoConfig.from_pretrained(
                pretrained_model_name_or_path, return_unused_kwargs=True, **kwargs
            )
590

591
592
593
594
        if type(config) in MODEL_MAPPING.keys():
            return MODEL_MAPPING[type(config)].from_pretrained(
                pretrained_model_name_or_path, *model_args, config=config, **kwargs
            )
595
        raise ValueError(
596
597
598
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_MAPPING.keys())
599
600
            )
        )
601
602


Julien Chaumond's avatar
Julien Chaumond committed
603
class AutoModelForPreTraining:
thomwolf's avatar
thomwolf committed
604
    r"""
605
606
607
608
    This is a generic model class that will be instantiated as one of the model classes of the library---with the
    architecture used for pretraining this model---when created with the when created with the
    :meth:`~transformers.AutoModelForPreTraining.from_pretrained` class method or the
    :meth:`~transformers.AutoModelForPreTraining.from_config` class method.
thomwolf's avatar
thomwolf committed
609

610
    This class cannot be instantiated directly using ``__init__()`` (throws an error).
thomwolf's avatar
thomwolf committed
611
612
613
614
615
616
617
618
619
620
    """

    def __init__(self):
        raise EnvironmentError(
            "AutoModelForPreTraining is designed to be instantiated "
            "using the `AutoModelForPreTraining.from_pretrained(pretrained_model_name_or_path)` or "
            "`AutoModelForPreTraining.from_config(config)` methods."
        )

    @classmethod
621
    @replace_list_option_in_docstrings(MODEL_FOR_PRETRAINING_MAPPING, use_model_types=False)
thomwolf's avatar
thomwolf committed
622
    def from_config(cls, config):
623
624
625
        r"""
        Instantiates one of the model classes of the library---with the architecture used for pretraining this
        model---from a configuration.
thomwolf's avatar
thomwolf committed
626

Lysandre's avatar
Lysandre committed
627
628
        Note:
            Loading a model from its configuration file does **not** load the model weights.
629
630
            It only affects the model's configuration. Use
            :meth:`~transformers.AutoModelForPreTraining.from_pretrained` to load the model weights.
Lysandre's avatar
Lysandre committed
631

thomwolf's avatar
thomwolf committed
632
633
634
635
        Args:
            config (:class:`~transformers.PretrainedConfig`):
                The model class to instantiate is selected based on the configuration class:

636
                List options
thomwolf's avatar
thomwolf committed
637
638
639

        Examples::

640
641
642
643
            >>> from transformers import AutoConfig, AutoModelForPreTraining
            >>> # Download configuration from S3 and cache.
            >>> config = AutoConfig.from_pretrained('bert-base-uncased')
            >>> model = AutoModelForPreTraining.from_config(config)
thomwolf's avatar
thomwolf committed
644
        """
645
646
        if type(config) in MODEL_FOR_PRETRAINING_MAPPING.keys():
            return MODEL_FOR_PRETRAINING_MAPPING[type(config)](config)
thomwolf's avatar
thomwolf committed
647
648
649
650
651
652
653
654
        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_FOR_PRETRAINING_MAPPING.keys())
            )
        )

    @classmethod
655
    @replace_list_option_in_docstrings(MODEL_FOR_PRETRAINING_MAPPING)
656
657
658
659
660
    @add_start_docstrings(
        "Instantiate one of the model classes of the library---with the architecture used for pretraining this ",
        "model---from a pretrained model.",
        AUTO_MODEL_PRETRAINED_DOCSTRING,
    )
thomwolf's avatar
thomwolf committed
661
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
662
663
        r"""
        Examples::
thomwolf's avatar
thomwolf committed
664

665
            >>> from transformers import AutoConfig, AutoModelForPreTraining
thomwolf's avatar
thomwolf committed
666

667
668
            >>> # Download model and configuration from S3 and cache.
            >>> model = AutoModelForPreTraining.from_pretrained('bert-base-uncased')
thomwolf's avatar
thomwolf committed
669

670
671
672
673
            >>> # Update configuration during loading
            >>> model = AutoModelForPreTraining.from_pretrained('bert-base-uncased', output_attentions=True)
            >>> model.config.output_attentions
            True
thomwolf's avatar
thomwolf committed
674

675
676
677
            >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            >>> config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
            >>> model = AutoModelForPreTraining.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
thomwolf's avatar
thomwolf committed
678
679
680
        """
        config = kwargs.pop("config", None)
        if not isinstance(config, PretrainedConfig):
681
682
683
            config, kwargs = AutoConfig.from_pretrained(
                pretrained_model_name_or_path, return_unused_kwargs=True, **kwargs
            )
thomwolf's avatar
thomwolf committed
684

685
686
687
688
        if type(config) in MODEL_FOR_PRETRAINING_MAPPING.keys():
            return MODEL_FOR_PRETRAINING_MAPPING[type(config)].from_pretrained(
                pretrained_model_name_or_path, *model_args, config=config, **kwargs
            )
thomwolf's avatar
thomwolf committed
689
690
691
692
693
694
695
696
        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_FOR_PRETRAINING_MAPPING.keys())
            )
        )


Julien Chaumond's avatar
Julien Chaumond committed
697
class AutoModelWithLMHead:
698
    r"""
699
700
701
702
703
704
    This is a generic model class that will be instantiated as one of the model classes of the library---with a
    language modeling head---when created with the when created with the
    :meth:`~transformers.AutoModelWithLMHead.from_pretrained` class method or the
    :meth:`~transformers.AutoModelWithLMHead.from_config` class method.

    This class cannot be instantiated directly using ``__init__()`` (throws an error).
705

706
707
708
709
710
711
    .. warning::

        This class is deprecated and will be removed in a future version. Please use
        :class:`~transformers.AutoModelForCausalLM` for causal language models,
        :class:`~transformers.AutoModelForMaskedLM` for masked language models and
        :class:`~transformers.AutoModelForSeq2SeqLM` for encoder-decoder models.
712
    """
713

714
    def __init__(self):
715
716
        raise EnvironmentError(
            "AutoModelWithLMHead is designed to be instantiated "
717
            "using the `AutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` or "
718
719
            "`AutoModelWithLMHead.from_config(config)` methods."
        )
720
721

    @classmethod
722
    @replace_list_option_in_docstrings(MODEL_WITH_LM_HEAD_MAPPING, use_model_types=False)
723
    def from_config(cls, config):
724
725
        r"""
        Instantiates one of the model classes of the library---with a language modeling head---from a configuration.
726

Lysandre's avatar
Lysandre committed
727
728
        Note:
            Loading a model from its configuration file does **not** load the model weights.
729
730
            It only affects the model's configuration. Use :meth:`~transformers.AutoModelWithLMHead.from_pretrained`
            to load the model weights.
Lysandre's avatar
Lysandre committed
731

Lysandre's avatar
Lysandre committed
732
733
        Args:
            config (:class:`~transformers.PretrainedConfig`):
734
                The model class to instantiate is selected based on the configuration class:
Lysandre's avatar
Lysandre committed
735

736
                List options
737
738
739

        Examples::

740
741
742
743
            >>> from transformers import AutoConfig, AutoModelWithLMHead
            >>> # Download configuration from S3 and cache.
            >>> config = AutoConfig.from_pretrained('bert-base-uncased')
            >>> model = AutoModelWithLMHead.from_config(config)
744
        """
745
        warnings.warn(
746
747
748
            "The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use "
            "`AutoModelForCausalLM` for causal language models, `AutoModelForMaskedLM` for masked language models and "
            "`AutoModelForSeq2SeqLM` for encoder-decoder models.",
749
750
            FutureWarning,
        )
751
752
        if type(config) in MODEL_WITH_LM_HEAD_MAPPING.keys():
            return MODEL_WITH_LM_HEAD_MAPPING[type(config)](config)
753
754
755
756
757
758
        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_WITH_LM_HEAD_MAPPING.keys())
            )
        )
759
760

    @classmethod
761
    @replace_list_option_in_docstrings(MODEL_WITH_LM_HEAD_MAPPING)
762
763
764
765
766
    @add_start_docstrings(
        "Instantiate one of the model classes of the library---with a language modeling head---from a pretrained ",
        "model.",
        AUTO_MODEL_PRETRAINED_DOCSTRING,
    )
767
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
768
769
        r"""
        Examples::
770

771
            >>> from transformers import AutoConfig, AutoModelWithLMHead
772

773
774
            >>> # Download model and configuration from S3 and cache.
            >>> model = AutoModelWithLMHead.from_pretrained('bert-base-uncased')
775

776
777
778
779
            >>> # Update configuration during loading
            >>> model = AutoModelWithLMHead.from_pretrained('bert-base-uncased', output_attentions=True)
            >>> model.config.output_attentions
            True
780

781
782
783
            >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            >>> config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
            >>> model = AutoModelWithLMHead.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
784
        """
785
        warnings.warn(
786
787
788
            "The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use "
            "`AutoModelForCausalLM` for causal language models, `AutoModelForMaskedLM` for masked language models and "
            "`AutoModelForSeq2SeqLM` for encoder-decoder models.",
789
790
            FutureWarning,
        )
791
792
        config = kwargs.pop("config", None)
        if not isinstance(config, PretrainedConfig):
793
794
795
            config, kwargs = AutoConfig.from_pretrained(
                pretrained_model_name_or_path, return_unused_kwargs=True, **kwargs
            )
796

797
798
799
800
        if type(config) in MODEL_WITH_LM_HEAD_MAPPING.keys():
            return MODEL_WITH_LM_HEAD_MAPPING[type(config)].from_pretrained(
                pretrained_model_name_or_path, *model_args, config=config, **kwargs
            )
801
        raise ValueError(
802
803
804
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_WITH_LM_HEAD_MAPPING.keys())
805
            )
806
807
808
809
810
        )


class AutoModelForCausalLM:
    r"""
811
812
813
814
    This is a generic model class that will be instantiated as one of the model classes of the library---with a
    causal language modeling head---when created with the when created with the
    :meth:`~transformers.AutoModelForCausalLM.from_pretrained` class method or the
    :meth:`~transformers.AutoModelForCausalLM.from_config` class method.
815

816
    This class cannot be instantiated directly using ``__init__()`` (throws an error).
817
818
819
820
821
822
823
824
825
826
    """

    def __init__(self):
        raise EnvironmentError(
            "AutoModelForCausalLM is designed to be instantiated "
            "using the `AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path)` or "
            "`AutoModelForCausalLM.from_config(config)` methods."
        )

    @classmethod
827
    @replace_list_option_in_docstrings(MODEL_FOR_CAUSAL_LM_MAPPING, use_model_types=False)
828
    def from_config(cls, config):
829
830
831
        r"""
        Instantiates one of the model classes of the library---with a causal language modeling head---from a
        configuration.
832
833
834

        Note:
            Loading a model from its configuration file does **not** load the model weights.
835
836
            It only affects the model's configuration. Use :meth:`~transformers.AutoModelForCausalLM.from_pretrained`
            to load the model weights.
837
838
839
840
841

        Args:
            config (:class:`~transformers.PretrainedConfig`):
                The model class to instantiate is selected based on the configuration class:

842
                List options
843
844
845

        Examples::

846
847
848
849
            >>> from transformers import AutoConfig, AutoModelForCausalLM
            >>> # Download configuration from S3 and cache.
            >>> config = AutoConfig.from_pretrained('gpt2')
            >>> model = AutoModelForCausalLM.from_config(config)
850
        """
851
852
        if type(config) in MODEL_FOR_CAUSAL_LM_MAPPING.keys():
            return MODEL_FOR_CAUSAL_LM_MAPPING[type(config)](config)
853
854
855
856
857
858
859
860
        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_FOR_CAUSAL_LM_MAPPING.keys())
            )
        )

    @classmethod
861
    @replace_list_option_in_docstrings(MODEL_FOR_CAUSAL_LM_MAPPING)
862
863
864
865
866
    @add_start_docstrings(
        "Instantiate one of the model classes of the library---with a causal language modeling head---from a "
        "pretrained model.",
        AUTO_MODEL_PRETRAINED_DOCSTRING,
    )
867
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
868
869
        r"""
        Examples::
870

871
            >>> from transformers import AutoConfig, AutoModelForCausalLM
872

873
874
            >>> # Download model and configuration from S3 and cache.
            >>> model = AutoModelForCausalLM.from_pretrained('gpt2')
875

876
877
878
879
            >>> # Update configuration during loading
            >>> model = AutoModelForCausalLM.from_pretrained('gpt2', output_attentions=True)
            >>> model.config.output_attentions
            True
880

881
882
883
            >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            >>> config = AutoConfig.from_json_file('./tf_model/gpt2_tf_model_config.json')
            >>> model = AutoModelForCausalLM.from_pretrained('./tf_model/gpt2_tf_checkpoint.ckpt.index', from_tf=True, config=config)
884
885
886
        """
        config = kwargs.pop("config", None)
        if not isinstance(config, PretrainedConfig):
887
888
889
            config, kwargs = AutoConfig.from_pretrained(
                pretrained_model_name_or_path, return_unused_kwargs=True, **kwargs
            )
890

891
892
893
894
        if type(config) in MODEL_FOR_CAUSAL_LM_MAPPING.keys():
            return MODEL_FOR_CAUSAL_LM_MAPPING[type(config)].from_pretrained(
                pretrained_model_name_or_path, *model_args, config=config, **kwargs
            )
895
896
897
898
899
900
901
902
903
904
        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_FOR_CAUSAL_LM_MAPPING.keys())
            )
        )


class AutoModelForMaskedLM:
    r"""
905
906
907
908
    This is a generic model class that will be instantiated as one of the model classes of the library---with a
    masked language modeling head---when created with the when created with the
    :meth:`~transformers.AutoModelForMaskedLM.from_pretrained` class method or the
    :meth:`~transformers.AutoModelForMasedLM.from_config` class method.
909

910
    This class cannot be instantiated directly using ``__init__()`` (throws an error).
911
912
913
914
915
916
917
918
919
920
    """

    def __init__(self):
        raise EnvironmentError(
            "AutoModelForMaskedLM is designed to be instantiated "
            "using the `AutoModelForMaskedLM.from_pretrained(pretrained_model_name_or_path)` or "
            "`AutoModelForMaskedLM.from_config(config)` methods."
        )

    @classmethod
921
    @replace_list_option_in_docstrings(MODEL_FOR_MASKED_LM_MAPPING, use_model_types=False)
922
    def from_config(cls, config):
923
924
925
        r"""
        Instantiates one of the model classes of the library---with a masked language modeling head---from a
        configuration.
926
927
928

        Note:
            Loading a model from its configuration file does **not** load the model weights.
929
930
            It only affects the model's configuration. Use :meth:`~transformers.AutoModelForMaskedLM.from_pretrained`
            to load the model weights.
931
932
933
934
935

        Args:
            config (:class:`~transformers.PretrainedConfig`):
                The model class to instantiate is selected based on the configuration class:

936
                List options
937
938
939

        Examples::

940
941
942
943
            >>> from transformers import AutoConfig, AutoModelForMaskedLM
            >>> # Download configuration from S3 and cache.
            >>> config = AutoConfig.from_pretrained('bert-base-uncased')
            >>> model = AutoModelForMaskedLM.from_config(config)
944
        """
945
946
        if type(config) in MODEL_FOR_MASKED_LM_MAPPING.keys():
            return MODEL_FOR_MASKED_LM_MAPPING[type(config)](config)
947
948
949
950
951
952
953
954
        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_FOR_MASKED_LM_MAPPING.keys())
            )
        )

    @classmethod
955
    @replace_list_option_in_docstrings(MODEL_FOR_MASKED_LM_MAPPING)
956
957
958
959
960
    @add_start_docstrings(
        "Instantiate one of the model classes of the library---with a masked language modeling head---from a "
        "pretrained model.",
        AUTO_MODEL_PRETRAINED_DOCSTRING,
    )
961
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
962
963
        r"""
        Examples::
964

965
            >>> from transformers import AutoConfig, AutoModelForMaskedLM
966

967
968
            >>> # Download model and configuration from S3 and cache.
            >>> model = AutoModelForMaskedLM.from_pretrained('bert-base-uncased')
969

970
971
972
973
            >>> # Update configuration during loading
            >>> model = AutoModelForMaskedLM.from_pretrained('bert-base-uncased', output_attentions=True)
            >>> model.config.output_attentions
            True
974

975
976
977
            >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            >>> config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
            >>> model = AutoModelForMaskedLM.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
978
979
980
        """
        config = kwargs.pop("config", None)
        if not isinstance(config, PretrainedConfig):
981
982
983
            config, kwargs = AutoConfig.from_pretrained(
                pretrained_model_name_or_path, return_unused_kwargs=True, **kwargs
            )
984

985
986
987
988
        if type(config) in MODEL_FOR_MASKED_LM_MAPPING.keys():
            return MODEL_FOR_MASKED_LM_MAPPING[type(config)].from_pretrained(
                pretrained_model_name_or_path, *model_args, config=config, **kwargs
            )
989
990
991
992
993
994
995
996
997
998
        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_FOR_MASKED_LM_MAPPING.keys())
            )
        )


class AutoModelForSeq2SeqLM:
    r"""
999
1000
1001
1002
    This is a generic model class that will be instantiated as one of the model classes of the library---with a
    sequence-to-sequence language modeling head---when created with the when created with the
    :meth:`~transformers.AutoModelForSeq2SeqLM.from_pretrained` class method or the
    :meth:`~transformers.AutoModelForSeq2SeqLM.from_config` class method.
1003

1004
    This class cannot be instantiated directly using ``__init__()`` (throws an error).
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
    """

    def __init__(self):
        raise EnvironmentError(
            "AutoModelForSeq2SeqLM is designed to be instantiated "
            "using the `AutoModelForSeq2SeqLM.from_pretrained(pretrained_model_name_or_path)` or "
            "`AutoModelForSeq2SeqLM.from_config(config)` methods."
        )

    @classmethod
1015
    @replace_list_option_in_docstrings(MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING, use_model_types=False)
1016
    def from_config(cls, config):
1017
1018
1019
        r"""
        Instantiates one of the model classes of the library---with a sequence-to-sequence language modeling
        head---from a configuration.
1020
1021
1022

        Note:
            Loading a model from its configuration file does **not** load the model weights.
1023
1024
            It only affects the model's configuration. Use :meth:`~transformers.AutoModelForSeq2SeqLM.from_pretrained`
            to load the model weights.
1025
1026
1027
1028
1029

        Args:
            config (:class:`~transformers.PretrainedConfig`):
                The model class to instantiate is selected based on the configuration class:

1030
                List options
1031
1032
1033

        Examples::

1034
1035
1036
1037
            >>> from transformers import AutoConfig, AutoModelForSeq2SeqLM
            >>> # Download configuration from S3 and cache.
            >>> config = AutoConfig.from_pretrained('t5')
            >>> model = AutoModelForSeq2SeqLM.from_config(config)
1038
        """
1039
1040
        if type(config) in MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.keys():
            return MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING[type(config)](config)
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__,
                cls.__name__,
                ", ".join(c.__name__ for c in MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.keys()),
            )
        )

    @classmethod
1051
    @replace_list_option_in_docstrings(MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING)
1052
1053
1054
1055
1056
    @add_start_docstrings(
        "Instantiate one of the model classes of the library---with a sequence-to-sequence language modeling "
        "head---from a pretrained model.",
        AUTO_MODEL_PRETRAINED_DOCSTRING,
    )
1057
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
1058
1059
        r"""
        Examples::
1060

1061
            >>> from transformers import AutoConfig, AutoModelForSeq2SeqLM
1062

1063
1064
            >>> # Download model and configuration from S3 and cache.
            >>> model = AutoModelForSeq2SeqLM.from_pretrained('t5-base')
1065

1066
1067
1068
1069
            >>> # Update configuration during loading
            >>> model = AutoModelForSeq2SeqLM.from_pretrained('t5-base', output_attentions=True)
            >>> model.config.output_attentions
            True
1070

1071
1072
1073
            >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            >>> config = AutoConfig.from_json_file('./tf_model/t5_tf_model_config.json')
            >>> model = AutoModelForSeq2SeqLM.from_pretrained('./tf_model/t5_tf_checkpoint.ckpt.index', from_tf=True, config=config)
1074
1075
1076
        """
        config = kwargs.pop("config", None)
        if not isinstance(config, PretrainedConfig):
1077
1078
1079
            config, kwargs = AutoConfig.from_pretrained(
                pretrained_model_name_or_path, return_unused_kwargs=True, **kwargs
            )
1080

1081
1082
1083
1084
        if type(config) in MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.keys():
            return MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING[type(config)].from_pretrained(
                pretrained_model_name_or_path, *model_args, config=config, **kwargs
            )
1085
1086
1087
1088
1089
1090
1091
        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__,
                cls.__name__,
                ", ".join(c.__name__ for c in MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.keys()),
            )
1092
        )
1093
1094


Julien Chaumond's avatar
Julien Chaumond committed
1095
class AutoModelForSequenceClassification:
1096
    r"""
1097
1098
1099
1100
    This is a generic model class that will be instantiated as one of the model classes of the library---with a
    sequence classification head---when created with the when created with the
    :meth:`~transformers.AutoModelForSequenceClassification.from_pretrained` class method or the
    :meth:`~transformers.AutoModelForSequenceClassification.from_config` class method.
1101

1102
    This class cannot be instantiated directly using ``__init__()`` (throws an error).
1103
    """
1104

1105
    def __init__(self):
1106
1107
        raise EnvironmentError(
            "AutoModelForSequenceClassification is designed to be instantiated "
1108
            "using the `AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path)` or "
1109
1110
            "`AutoModelForSequenceClassification.from_config(config)` methods."
        )
1111
1112

    @classmethod
1113
    @replace_list_option_in_docstrings(MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING, use_model_types=False)
1114
    def from_config(cls, config):
1115
1116
1117
        r"""
        Instantiates one of the model classes of the library---with a sequence classification head---from a
        configuration.
1118

Lysandre's avatar
Lysandre committed
1119
1120
        Note:
            Loading a model from its configuration file does **not** load the model weights.
1121
1122
            It only affects the model's configuration. Use
            :meth:`~transformers.AutoModelForSequenceClassification.from_pretrained` to load the model weights.
Lysandre's avatar
Lysandre committed
1123

Lysandre's avatar
Lysandre committed
1124
1125
        Args:
            config (:class:`~transformers.PretrainedConfig`):
1126
                The model class to instantiate is selected based on the configuration class:
Lysandre's avatar
Lysandre committed
1127

1128
                List options
1129
1130
1131

        Examples::

1132
1133
1134
1135
            >>> from transformers import AutoConfig, AutoModelForSequenceClassification
            >>> # Download configuration from S3 and cache.
            >>> config = AutoConfig.from_pretrained('bert-base-uncased')
            >>> model = AutoModelForSequenceClassification.from_config(config)
1136
        """
1137
1138
        if type(config) in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.keys():
            return MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING[type(config)](config)
1139
1140
1141
1142
1143
1144
1145
1146
        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__,
                cls.__name__,
                ", ".join(c.__name__ for c in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.keys()),
            )
        )
1147
1148

    @classmethod
1149
    @replace_list_option_in_docstrings(MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING)
1150
1151
1152
1153
1154
    @add_start_docstrings(
        "Instantiate one of the model classes of the library---with a sequence classification head---from a "
        "pretrained model.",
        AUTO_MODEL_PRETRAINED_DOCSTRING,
    )
1155
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
1156
1157
        r"""
        Examples::
thomwolf's avatar
thomwolf committed
1158

1159
            >>> from transformers import AutoConfig, AutoModelForSequenceClassification
1160

1161
1162
            >>> # Download model and configuration from S3 and cache.
            >>> model = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased')
1163

1164
1165
1166
1167
            >>> # Update configuration during loading
            >>> model = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased', output_attentions=True)
            >>> model.config.output_attentions
            True
1168

1169
1170
1171
            >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            >>> config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
            >>> model = AutoModelForSequenceClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
1172
        """
1173
1174
        config = kwargs.pop("config", None)
        if not isinstance(config, PretrainedConfig):
1175
1176
1177
            config, kwargs = AutoConfig.from_pretrained(
                pretrained_model_name_or_path, return_unused_kwargs=True, **kwargs
            )
1178

1179
1180
1181
1182
        if type(config) in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.keys():
            return MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING[type(config)].from_pretrained(
                pretrained_model_name_or_path, *model_args, config=config, **kwargs
            )
1183
        raise ValueError(
1184
1185
1186
1187
1188
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__,
                cls.__name__,
                ", ".join(c.__name__ for c in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.keys()),
1189
1190
            )
        )
1191
1192


Julien Chaumond's avatar
Julien Chaumond committed
1193
class AutoModelForQuestionAnswering:
1194
    r"""
1195
1196
1197
1198
    This is a generic model class that will be instantiated as one of the model classes of the library---with a
    question answering head---when created with the when created with the
    :meth:`~transformers.AutoModeForQuestionAnswering.from_pretrained` class method or the
    :meth:`~transformers.AutoModelForQuestionAnswering.from_config` class method.
1199

1200
    This class cannot be instantiated directly using ``__init__()`` (throws an error).
1201
    """
1202

1203
    def __init__(self):
1204
1205
        raise EnvironmentError(
            "AutoModelForQuestionAnswering is designed to be instantiated "
1206
            "using the `AutoModelForQuestionAnswering.from_pretrained(pretrained_model_name_or_path)` or "
1207
1208
            "`AutoModelForQuestionAnswering.from_config(config)` methods."
        )
1209
1210

    @classmethod
1211
    @replace_list_option_in_docstrings(MODEL_FOR_QUESTION_ANSWERING_MAPPING, use_model_types=False)
1212
    def from_config(cls, config):
1213
1214
        r"""
        Instantiates one of the model classes of the library---with a question answering head---from a configuration.
1215

Lysandre's avatar
Lysandre committed
1216
1217
        Note:
            Loading a model from its configuration file does **not** load the model weights.
1218
1219
            It only affects the model's configuration. Use
            :meth:`~transformers.AutoModelForQuestionAnswering.from_pretrained` to load the model weights.
Lysandre's avatar
Lysandre committed
1220

Lysandre's avatar
Lysandre committed
1221
1222
        Args:
            config (:class:`~transformers.PretrainedConfig`):
1223
                The model class to instantiate is selected based on the configuration class:
Lysandre's avatar
Lysandre committed
1224

1225
                List options
1226
1227
1228

        Examples::

1229
1230
1231
1232
            >>> from transformers import AutoConfig, AutoModelForQuestionAnswering
            >>> # Download configuration from S3 and cache.
            >>> config = AutoConfig.from_pretrained('bert-base-uncased')
            >>> model = AutoModelForQuestionAnswering.from_config(config)
1233
        """
1234
1235
        if type(config) in MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys():
            return MODEL_FOR_QUESTION_ANSWERING_MAPPING[type(config)](config)
1236
1237
1238
1239
1240
1241
1242
1243
1244

        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__,
                cls.__name__,
                ", ".join(c.__name__ for c in MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys()),
            )
        )
1245
1246

    @classmethod
1247
    @replace_list_option_in_docstrings(MODEL_FOR_QUESTION_ANSWERING_MAPPING)
1248
1249
1250
1251
1252
    @add_start_docstrings(
        "Instantiate one of the model classes of the library---with a question answering head---from a "
        "pretrained model.",
        AUTO_MODEL_PRETRAINED_DOCSTRING,
    )
1253
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
1254
1255
        r"""
        Examples::
thomwolf's avatar
thomwolf committed
1256

1257
            >>> from transformers import AutoConfig, AutoModelForQuestionAnswering
1258

1259
1260
            >>> # Download model and configuration from S3 and cache.
            >>> model = AutoModelForQuestionAnswering.from_pretrained('bert-base-uncased')
1261

1262
1263
1264
1265
            >>> # Update configuration during loading
            >>> model = AutoModelForQuestionAnswering.from_pretrained('bert-base-uncased', output_attentions=True)
            >>> model.config.output_attentions
            True
1266

1267
1268
1269
            >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            >>> config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
            >>> model = AutoModelForQuestionAnswering.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
1270
        """
1271
1272
        config = kwargs.pop("config", None)
        if not isinstance(config, PretrainedConfig):
1273
1274
1275
            config, kwargs = AutoConfig.from_pretrained(
                pretrained_model_name_or_path, return_unused_kwargs=True, **kwargs
            )
1276

1277
1278
1279
1280
        if type(config) in MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys():
            return MODEL_FOR_QUESTION_ANSWERING_MAPPING[type(config)].from_pretrained(
                pretrained_model_name_or_path, *model_args, config=config, **kwargs
            )
1281

1282
        raise ValueError(
1283
1284
1285
1286
1287
1288
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__,
                cls.__name__,
                ", ".join(c.__name__ for c in MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys()),
            )
1289
        )
1290
1291
1292


class AutoModelForTokenClassification:
Lysandre's avatar
Lysandre committed
1293
    r"""
1294
1295
1296
1297
    This is a generic model class that will be instantiated as one of the model classes of the library---with a
    token classification head---when created with the when created with the
    :meth:`~transformers.AutoModelForTokenClassification.from_pretrained` class method or the
    :meth:`~transformers.AutoModelForTokenClassification.from_config` class method.
Lysandre's avatar
Lysandre committed
1298

1299
    This class cannot be instantiated directly using ``__init__()`` (throws an error).
Lysandre's avatar
Lysandre committed
1300
1301
    """

1302
    def __init__(self):
1303
1304
1305
1306
1307
        raise EnvironmentError(
            "AutoModelForTokenClassification is designed to be instantiated "
            "using the `AutoModelForTokenClassification.from_pretrained(pretrained_model_name_or_path)` or "
            "`AutoModelForTokenClassification.from_config(config)` methods."
        )
1308
1309

    @classmethod
1310
    @replace_list_option_in_docstrings(MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING, use_model_types=False)
1311
    def from_config(cls, config):
1312
1313
        r"""
        Instantiates one of the model classes of the library---with a token classification head---from a configuration.
1314

Lysandre's avatar
Lysandre committed
1315
1316
        Note:
            Loading a model from its configuration file does **not** load the model weights.
1317
1318
            It only affects the model's configuration. Use
            :meth:`~transformers.AutoModelForTokenClassification.from_pretrained` to load the model weights.
Lysandre's avatar
Lysandre committed
1319

Lysandre's avatar
Lysandre committed
1320
1321
        Args:
            config (:class:`~transformers.PretrainedConfig`):
1322
                The model class to instantiate is selected based on the configuration class:
Lysandre's avatar
Lysandre committed
1323

1324
                List options
1325

1326
        Examples::
1327

1328
1329
1330
1331
            >>> from transformers import AutoConfig, AutoModelForTokenClassification
            >>> # Download configuration from S3 and cache.
            >>> config = AutoConfig.from_pretrained('bert-base-uncased')
            >>> model = AutoModelForTokenClassification.from_config(config)
1332
        """
1333
1334
        if type(config) in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys():
            return MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING[type(config)](config)
1335
1336
1337
1338
1339
1340
1341
1342
1343

        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__,
                cls.__name__,
                ", ".join(c.__name__ for c in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys()),
            )
        )
1344

1345
    @classmethod
1346
    @replace_list_option_in_docstrings(MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING)
1347
1348
1349
1350
1351
    @add_start_docstrings(
        "Instantiate one of the model classes of the library---with a token classification head---from a "
        "pretrained model.",
        AUTO_MODEL_PRETRAINED_DOCSTRING,
    )
1352
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
1353
1354
        r"""
        Examples::
1355

1356
            >>> from transformers import AutoConfig, AutoModelForTokenClassification
1357

1358
1359
            >>> # Download model and configuration from S3 and cache.
            >>> model = AutoModelForTokenClassification.from_pretrained('bert-base-uncased')
1360

1361
1362
1363
1364
            >>> # Update configuration during loading
            >>> model = AutoModelForTokenClassification.from_pretrained('bert-base-uncased', output_attentions=True)
            >>> model.config.output_attentions
            True
1365

1366
1367
1368
            >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            >>> config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
            >>> model = AutoModelForTokenClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
1369
        """
1370
1371
        config = kwargs.pop("config", None)
        if not isinstance(config, PretrainedConfig):
1372
1373
1374
            config, kwargs = AutoConfig.from_pretrained(
                pretrained_model_name_or_path, return_unused_kwargs=True, **kwargs
            )
1375

1376
1377
1378
1379
        if type(config) in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys():
            return MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING[type(config)].from_pretrained(
                pretrained_model_name_or_path, *model_args, config=config, **kwargs
            )
1380

1381
        raise ValueError(
1382
1383
1384
1385
1386
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__,
                cls.__name__,
                ", ".join(c.__name__ for c in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys()),
1387
1388
            )
        )
Julien Chaumond's avatar
Julien Chaumond committed
1389
1390
1391
1392


class AutoModelForMultipleChoice:
    r"""
1393
1394
1395
1396
    This is a generic model class that will be instantiated as one of the model classes of the library---with a
    multiple choice classifcation head---when created with the when created with the
    :meth:`~transformers.AutoModelForMultipleChoice.from_pretrained` class method or the
    :meth:`~transformers.AutoModelForMultipleChoice.from_config` class method.
Julien Chaumond's avatar
Julien Chaumond committed
1397

1398
    This class cannot be instantiated directly using ``__init__()`` (throws an error).
Julien Chaumond's avatar
Julien Chaumond committed
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
    """

    def __init__(self):
        raise EnvironmentError(
            "AutoModelForMultipleChoice is designed to be instantiated "
            "using the `AutoModelForMultipleChoice.from_pretrained(pretrained_model_name_or_path)` or "
            "`AutoModelForMultipleChoice.from_config(config)` methods."
        )

    @classmethod
1409
    @replace_list_option_in_docstrings(MODEL_FOR_MULTIPLE_CHOICE_MAPPING, use_model_types=False)
Julien Chaumond's avatar
Julien Chaumond committed
1410
    def from_config(cls, config):
1411
1412
1413
        r"""
        Instantiates one of the model classes of the library---with a multiple choice classification head---from a
        configuration.
1414
1415
1416

        Note:
            Loading a model from its configuration file does **not** load the model weights.
1417
1418
            It only affects the model's configuration. Use
            :meth:`~transformers.AutoModelForMultipleChoice.from_pretrained` to load the model weights.
1419
1420
1421
1422
1423
1424
1425
1426
1427

        Args:
            config (:class:`~transformers.PretrainedConfig`):
                The model class to instantiate is selected based on the configuration class:

                List options

        Examples::

1428
1429
1430
1431
            >>> from transformers import AutoConfig, AutoModelForMultipleChoice
            >>> # Download configuration from S3 and cache.
            >>> config = AutoConfig.from_pretrained('bert-base-uncased')
            >>> model = AutoModelForMultipleChoice.from_config(config)
1432
        """
1433
1434
        if type(config) in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.keys():
            return MODEL_FOR_MULTIPLE_CHOICE_MAPPING[type(config)](config)
Julien Chaumond's avatar
Julien Chaumond committed
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445

        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__,
                cls.__name__,
                ", ".join(c.__name__ for c in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.keys()),
            )
        )

    @classmethod
1446
    @replace_list_option_in_docstrings(MODEL_FOR_MULTIPLE_CHOICE_MAPPING)
1447
1448
1449
1450
1451
    @add_start_docstrings(
        "Instantiate one of the model classes of the library---with a multiple choice classification head---from a "
        "pretrained model.",
        AUTO_MODEL_PRETRAINED_DOCSTRING,
    )
Julien Chaumond's avatar
Julien Chaumond committed
1452
    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
1453
1454
        r"""
        Examples::
1455

1456
            >>> from transformers import AutoConfig, AutoModelForMultipleChoice
1457

1458
1459
            >>> # Download model and configuration from S3 and cache.
            >>> model = AutoModelForMultipleChoice.from_pretrained('bert-base-uncased')
1460

1461
1462
1463
1464
            >>> # Update configuration during loading
            >>> model = AutoModelForMultipleChoice.from_pretrained('bert-base-uncased', output_attentions=True)
            >>> model.config.output_attentions
            True
1465

1466
1467
1468
            >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            >>> config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
            >>> model = AutoModelForMultipleChoice.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
1469
        """
Julien Chaumond's avatar
Julien Chaumond committed
1470
1471
        config = kwargs.pop("config", None)
        if not isinstance(config, PretrainedConfig):
1472
1473
1474
            config, kwargs = AutoConfig.from_pretrained(
                pretrained_model_name_or_path, return_unused_kwargs=True, **kwargs
            )
Julien Chaumond's avatar
Julien Chaumond committed
1475

1476
1477
1478
1479
        if type(config) in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.keys():
            return MODEL_FOR_MULTIPLE_CHOICE_MAPPING[type(config)].from_pretrained(
                pretrained_model_name_or_path, *model_args, config=config, **kwargs
            )
Julien Chaumond's avatar
Julien Chaumond committed
1480
1481
1482
1483
1484
1485
1486
1487
1488

        raise ValueError(
            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
            "Model type should be one of {}.".format(
                config.__class__,
                cls.__name__,
                ", ".join(c.__name__ for c in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.keys()),
            )
        )