__init__.py 20.6 KB
Newer Older
1
2
3
4
# flake8: noqa
# There's no way to ignore "F401 '...' imported but unused" warnings in this
# module, but to preserve other warnings. So, don't check this module at all.

Lysandre's avatar
Lysandre committed
5
__version__ = "3.0.0"
thomwolf's avatar
thomwolf committed
6

thomwolf's avatar
thomwolf committed
7
8
9
10
11
12
# Work around to update TensorFlow's absl.logging threshold which alters the
# default Python logging output behavior when present.
# see: https://github.com/abseil/abseil-py/issues/99
# and: https://github.com/tensorflow/tensorflow/issues/26691#issuecomment-500369493
try:
    import absl.logging
13
14
15
except ImportError:
    pass
else:
16
17
    absl.logging.set_verbosity("info")
    absl.logging.set_stderrthreshold("info")
thomwolf's avatar
thomwolf committed
18
    absl.logging._warn_preinit_stderr = False
19

thomwolf's avatar
thomwolf committed
20
21
import logging

Julien Chaumond's avatar
Julien Chaumond committed
22
# Configurations
Aymeric Augustin's avatar
Aymeric Augustin committed
23
from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig
24
from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, CONFIG_MAPPING, AutoConfig
Sam Shleifer's avatar
Sam Shleifer committed
25
from .configuration_bart import BartConfig
Aymeric Augustin's avatar
Aymeric Augustin committed
26
27
28
29
from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig
from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
from .configuration_distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig
Lysandre Debut's avatar
Lysandre Debut committed
30
from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig
31
from .configuration_encoder_decoder import EncoderDecoderConfig
Hang Le's avatar
Hang Le committed
32
from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig
Aymeric Augustin's avatar
Aymeric Augustin committed
33
from .configuration_gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config
Iz Beltagy's avatar
Iz Beltagy committed
34
from .configuration_longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig
35
from .configuration_marian import MarianConfig
Aymeric Augustin's avatar
Aymeric Augustin committed
36
from .configuration_mmbt import MMBTConfig
Vasily Shamporov's avatar
Vasily Shamporov committed
37
from .configuration_mobilebert import MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileBertConfig
Aymeric Augustin's avatar
Aymeric Augustin committed
38
from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
Patrick von Platen's avatar
Patrick von Platen committed
39
from .configuration_reformer import REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, ReformerConfig
Yacine Jernite's avatar
Yacine Jernite committed
40
from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
Aymeric Augustin's avatar
Aymeric Augustin committed
41
42
43
44
45
46
47
from .configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig
from .configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config
from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
from .configuration_utils import PretrainedConfig
from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig
from .configuration_xlm_roberta import XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMRobertaConfig
from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig
48
from .data import (
Aymeric Augustin's avatar
Aymeric Augustin committed
49
    DataProcessor,
50
51
52
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
Aymeric Augustin's avatar
Aymeric Augustin committed
53
54
55
56
    SquadExample,
    SquadFeatures,
    SquadV1Processor,
    SquadV2Processor,
57
    glue_convert_examples_to_features,
Aymeric Augustin's avatar
Aymeric Augustin committed
58
    glue_output_modes,
59
60
    glue_processors,
    glue_tasks_num_labels,
Aymeric Augustin's avatar
Aymeric Augustin committed
61
62
    is_sklearn_available,
    squad_convert_examples_to_features,
63
64
65
66
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)
67

Aymeric Augustin's avatar
Aymeric Augustin committed
68
69
70
71
72
73
74
75
76
77
78
79
80
# Files and general utilities
from .file_utils import (
    CONFIG_NAME,
    MODEL_CARD_NAME,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    add_end_docstrings,
    add_start_docstrings,
    cached_path,
Patrick von Platen's avatar
Patrick von Platen committed
81
82
83
    is_apex_available,
    is_psutil_available,
    is_py3nvml_available,
Aymeric Augustin's avatar
Aymeric Augustin committed
84
85
    is_tf_available,
    is_torch_available,
86
    is_torch_tpu_available,
Aymeric Augustin's avatar
Aymeric Augustin committed
87
)
88
from .hf_argparser import HfArgumentParser
89

thomwolf's avatar
thomwolf committed
90
# Model Cards
91
from .modelcard import ModelCard
92

Aymeric Augustin's avatar
Aymeric Augustin committed
93
94
95
96
97
98
99
100
101
102
# TF 2.0 <=> PyTorch conversion utilities
from .modeling_tf_pytorch_utils import (
    convert_tf_weight_name_to_pt_weight_name,
    load_pytorch_checkpoint_in_tf2_model,
    load_pytorch_model_in_tf2_model,
    load_pytorch_weights_in_tf2_model,
    load_tf2_checkpoint_in_pytorch_model,
    load_tf2_model_in_pytorch_model,
    load_tf2_weights_in_pytorch_model,
)
103

Aymeric Augustin's avatar
Aymeric Augustin committed
104
105
106
107
# Pipelines
from .pipelines import (
    CsvPipelineDataFormat,
    FeatureExtractionPipeline,
Julien Chaumond's avatar
Julien Chaumond committed
108
    FillMaskPipeline,
Aymeric Augustin's avatar
Aymeric Augustin committed
109
110
111
112
113
114
    JsonPipelineDataFormat,
    NerPipeline,
    PipedPipelineDataFormat,
    Pipeline,
    PipelineDataFormat,
    QuestionAnsweringPipeline,
115
    SummarizationPipeline,
Aymeric Augustin's avatar
Aymeric Augustin committed
116
    TextClassificationPipeline,
117
    TextGenerationPipeline,
118
    TokenClassificationPipeline,
119
    TranslationPipeline,
Aymeric Augustin's avatar
Aymeric Augustin committed
120
121
    pipeline,
)
Julien Chaumond's avatar
Julien Chaumond committed
122
123

# Tokenizers
Aymeric Augustin's avatar
Aymeric Augustin committed
124
from .tokenization_albert import AlbertTokenizer
125
from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer
Suraj Patil's avatar
Suraj Patil committed
126
from .tokenization_bart import BartTokenizer, BartTokenizerFast, MBartTokenizer
Anthony MOI's avatar
Anthony MOI committed
127
from .tokenization_bert import BasicTokenizer, BertTokenizer, BertTokenizerFast, WordpieceTokenizer
Aymeric Augustin's avatar
Aymeric Augustin committed
128
129
from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer
from .tokenization_camembert import CamembertTokenizer
keskarnitish's avatar
keskarnitish committed
130
from .tokenization_ctrl import CTRLTokenizer
131
from .tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFast
Lysandre Debut's avatar
Lysandre Debut committed
132
from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast
Hang Le's avatar
Hang Le committed
133
from .tokenization_flaubert import FlaubertTokenizer
Anthony MOI's avatar
Anthony MOI committed
134
from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
Suraj Patil's avatar
Suraj Patil committed
135
from .tokenization_longformer import LongformerTokenizer, LongformerTokenizerFast
Vasily Shamporov's avatar
Vasily Shamporov committed
136
from .tokenization_mobilebert import MobileBertTokenizer, MobileBertTokenizerFast
137
from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
Patrick von Platen's avatar
Patrick von Platen committed
138
from .tokenization_reformer import ReformerTokenizer
Yacine Jernite's avatar
Yacine Jernite committed
139
from .tokenization_retribert import RetriBertTokenizer, RetriBertTokenizerFast
140
from .tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
thomwolf's avatar
thomwolf committed
141
from .tokenization_t5 import T5Tokenizer
142
from .tokenization_transfo_xl import TransfoXLCorpus, TransfoXLTokenizer, TransfoXLTokenizerFast
143
144
from .tokenization_utils import PreTrainedTokenizer
from .tokenization_utils_base import (
145
    BatchEncoding,
146
147
    CharSpan,
    PreTrainedTokenizerBase,
148
149
    SpecialTokensMixin,
    TensorType,
150
    TokenSpan,
151
)
152
from .tokenization_utils_fast import PreTrainedTokenizerFast
Aymeric Augustin's avatar
Aymeric Augustin committed
153
from .tokenization_xlm import XLMTokenizer
154
from .tokenization_xlm_roberta import XLMRobertaTokenizer
Aymeric Augustin's avatar
Aymeric Augustin committed
155
from .tokenization_xlnet import SPIECE_UNDERLINE, XLNetTokenizer
156
157

# Trainer
158
from .trainer_utils import EvalPrediction, set_seed
159
from .training_args import TrainingArguments
Julien Plu's avatar
Julien Plu committed
160
from .training_args_tf import TFTrainingArguments
Aymeric Augustin's avatar
Aymeric Augustin committed
161
162
163
164
165
166
167


logger = logging.getLogger(__name__)  # pylint: disable=invalid-name


if is_sklearn_available():
    from .data import glue_compute_metrics, xnli_compute_metrics
168
169


170
# Modeling
thomwolf's avatar
thomwolf committed
171
if is_torch_available():
172
173
    from .generation_utils import top_k_top_p_filtering
    from .modeling_utils import PreTrainedModel, prune_layer, Conv1D, apply_chunking_to_forward
174
175
    from .modeling_auto import (
        AutoModel,
thomwolf's avatar
thomwolf committed
176
        AutoModelForPreTraining,
177
178
179
        AutoModelForSequenceClassification,
        AutoModelForQuestionAnswering,
        AutoModelWithLMHead,
180
181
182
        AutoModelForCausalLM,
        AutoModelForMaskedLM,
        AutoModelForSeq2SeqLM,
183
        AutoModelForTokenClassification,
Julien Chaumond's avatar
Julien Chaumond committed
184
        AutoModelForMultipleChoice,
185
186
187
        MODEL_MAPPING,
        MODEL_FOR_PRETRAINING_MAPPING,
        MODEL_WITH_LM_HEAD_MAPPING,
188
189
190
        MODEL_FOR_CAUSAL_LM_MAPPING,
        MODEL_FOR_MASKED_LM_MAPPING,
        MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
191
192
193
        MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
        MODEL_FOR_QUESTION_ANSWERING_MAPPING,
        MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
Julien Chaumond's avatar
Julien Chaumond committed
194
        MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
195
196
    )

Vasily Shamporov's avatar
Vasily Shamporov committed
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
    from .modeling_mobilebert import (
        MobileBertPreTrainedModel,
        MobileBertModel,
        MobileBertForPreTraining,
        MobileBertForSequenceClassification,
        MobileBertForQuestionAnswering,
        MobileBertForMaskedLM,
        MobileBertForNextSentencePrediction,
        MobileBertForMultipleChoice,
        MobileBertForTokenClassification,
        load_tf_weights_in_mobilebert,
        MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        MobileBertLayer,
    )

212
213
214
215
216
    from .modeling_bert import (
        BertPreTrainedModel,
        BertModel,
        BertForPreTraining,
        BertForMaskedLM,
217
        BertLMHeadModel,
218
219
220
221
222
223
        BertForNextSentencePrediction,
        BertForSequenceClassification,
        BertForMultipleChoice,
        BertForTokenClassification,
        BertForQuestionAnswering,
        load_tf_weights_in_bert,
224
        BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
Patrick von Platen's avatar
Patrick von Platen committed
225
        BertLayer,
226
227
228
229
230
231
232
    )
    from .modeling_openai import (
        OpenAIGPTPreTrainedModel,
        OpenAIGPTModel,
        OpenAIGPTLMHeadModel,
        OpenAIGPTDoubleHeadsModel,
        load_tf_weights_in_openai_gpt,
233
        OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
234
235
236
237
238
239
240
    )
    from .modeling_transfo_xl import (
        TransfoXLPreTrainedModel,
        TransfoXLModel,
        TransfoXLLMHeadModel,
        AdaptiveEmbedding,
        load_tf_weights_in_transfo_xl,
241
        TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
242
243
244
245
246
247
248
    )
    from .modeling_gpt2 import (
        GPT2PreTrainedModel,
        GPT2Model,
        GPT2LMHeadModel,
        GPT2DoubleHeadsModel,
        load_tf_weights_in_gpt2,
249
        GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
250
    )
251
    from .modeling_ctrl import CTRLPreTrainedModel, CTRLModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_LIST
252
253
254
255
256
257
258
259
260
261
    from .modeling_xlnet import (
        XLNetPreTrainedModel,
        XLNetModel,
        XLNetLMHeadModel,
        XLNetForSequenceClassification,
        XLNetForTokenClassification,
        XLNetForMultipleChoice,
        XLNetForQuestionAnsweringSimple,
        XLNetForQuestionAnswering,
        load_tf_weights_in_xlnet,
262
        XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
263
264
265
266
267
268
    )
    from .modeling_xlm import (
        XLMPreTrainedModel,
        XLMModel,
        XLMWithLMHeadModel,
        XLMForSequenceClassification,
269
        XLMForTokenClassification,
270
271
        XLMForQuestionAnswering,
        XLMForQuestionAnsweringSimple,
272
        XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
273
    )
274
    from .modeling_bart import (
275
        PretrainedBartModel,
276
277
278
        BartForSequenceClassification,
        BartModel,
        BartForConditionalGeneration,
Suraj Patil's avatar
Suraj Patil committed
279
        BartForQuestionAnswering,
280
        BART_PRETRAINED_MODEL_ARCHIVE_LIST,
281
    )
282
    from .modeling_marian import MarianMTModel
283
    from .tokenization_marian import MarianTokenizer
284
285
286
287
288
289
290
    from .modeling_roberta import (
        RobertaForMaskedLM,
        RobertaModel,
        RobertaForSequenceClassification,
        RobertaForMultipleChoice,
        RobertaForTokenClassification,
        RobertaForQuestionAnswering,
291
        ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
292
293
294
295
296
    )
    from .modeling_distilbert import (
        DistilBertPreTrainedModel,
        DistilBertForMaskedLM,
        DistilBertModel,
297
        DistilBertForMultipleChoice,
298
299
300
        DistilBertForSequenceClassification,
        DistilBertForQuestionAnswering,
        DistilBertForTokenClassification,
301
        DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
302
303
304
305
306
307
308
    )
    from .modeling_camembert import (
        CamembertForMaskedLM,
        CamembertModel,
        CamembertForSequenceClassification,
        CamembertForMultipleChoice,
        CamembertForTokenClassification,
309
        CamembertForQuestionAnswering,
310
        CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
311
    )
312
    from .modeling_encoder_decoder import EncoderDecoderModel
313
314
315
    from .modeling_t5 import (
        T5PreTrainedModel,
        T5Model,
316
        T5ForConditionalGeneration,
317
        load_tf_weights_in_t5,
318
        T5_PRETRAINED_MODEL_ARCHIVE_LIST,
319
320
321
322
    )
    from .modeling_albert import (
        AlbertPreTrainedModel,
        AlbertModel,
323
        AlbertForPreTraining,
324
        AlbertForMaskedLM,
325
        AlbertForMultipleChoice,
326
327
        AlbertForSequenceClassification,
        AlbertForQuestionAnswering,
328
        AlbertForTokenClassification,
329
        load_tf_weights_in_albert,
330
        ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
331
332
333
334
335
336
337
    )
    from .modeling_xlm_roberta import (
        XLMRobertaForMaskedLM,
        XLMRobertaModel,
        XLMRobertaForMultipleChoice,
        XLMRobertaForSequenceClassification,
        XLMRobertaForTokenClassification,
338
        XLMRobertaForQuestionAnswering,
339
        XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
340
    )
341
342
    from .modeling_mmbt import ModalEmbeddings, MMBTModel, MMBTForClassification

Hang Le's avatar
Hang Le committed
343
344
345
346
347
348
    from .modeling_flaubert import (
        FlaubertModel,
        FlaubertWithLMHeadModel,
        FlaubertForSequenceClassification,
        FlaubertForQuestionAnswering,
        FlaubertForQuestionAnsweringSimple,
349
        FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
Hang Le's avatar
Hang Le committed
350
351
    )

Lysandre Debut's avatar
Lysandre Debut committed
352
353
354
355
    from .modeling_electra import (
        ElectraForPreTraining,
        ElectraForMaskedLM,
        ElectraForTokenClassification,
356
        ElectraPreTrainedModel,
Suraj Patil's avatar
Suraj Patil committed
357
        ElectraForMultipleChoice,
358
        ElectraForSequenceClassification,
359
        ElectraForQuestionAnswering,
Lysandre Debut's avatar
Lysandre Debut committed
360
361
        ElectraModel,
        load_tf_weights_in_electra,
362
        ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
Lysandre Debut's avatar
Lysandre Debut committed
363
364
    )

Patrick von Platen's avatar
Patrick von Platen committed
365
366
367
368
369
    from .modeling_reformer import (
        ReformerAttention,
        ReformerLayer,
        ReformerModel,
        ReformerModelWithLMHead,
370
        ReformerForQuestionAnswering,
371
        REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
Patrick von Platen's avatar
Patrick von Platen committed
372
373
    )

374
375
376
    from .modeling_longformer import (
        LongformerModel,
        LongformerForMaskedLM,
377
        LongformerForSequenceClassification,
378
        LongformerForMultipleChoice,
379
        LongformerForTokenClassification,
380
        LongformerForQuestionAnswering,
381
        LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
382
    )
Iz Beltagy's avatar
Iz Beltagy committed
383

Yacine Jernite's avatar
Yacine Jernite committed
384
385
386
387
388
389
    from .modeling_retribert import (
        RetriBertPreTrainedModel,
        RetriBertModel,
        RETRIBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
    )

thomwolf's avatar
thomwolf committed
390
    # Optimization
391
392
393
394
395
396
397
398
    from .optimization import (
        AdamW,
        get_constant_schedule,
        get_constant_schedule_with_warmup,
        get_cosine_schedule_with_warmup,
        get_cosine_with_hard_restarts_schedule_with_warmup,
        get_linear_schedule_with_warmup,
    )
thomwolf's avatar
thomwolf committed
399

Julien Chaumond's avatar
Julien Chaumond committed
400
    # Trainer
401
    from .trainer import Trainer, torch_distributed_zero_first
402
    from .data.data_collator import default_data_collator, DataCollator, DataCollatorForLanguageModeling
Julien Chaumond's avatar
Julien Chaumond committed
403
    from .data.datasets import GlueDataset, TextDataset, LineByLineTextDataset, GlueDataTrainingArguments
thomwolf's avatar
thomwolf committed
404

405
    # Benchmarks
Patrick von Platen's avatar
Patrick von Platen committed
406
407
    from .benchmark.benchmark import PyTorchBenchmark
    from .benchmark.benchmark_args import PyTorchBenchmarkArguments
408

thomwolf's avatar
thomwolf committed
409
# TensorFlow
thomwolf's avatar
thomwolf committed
410
if is_tf_available():
411
    from .generation_tf_utils import tf_top_k_top_p_filtering
412
413
    from .modeling_tf_utils import (
        shape_list,
Julien Plu's avatar
Julien Plu committed
414
415
416
        TFPreTrainedModel,
        TFSequenceSummary,
        TFSharedEmbeddings,
417
    )
418
    from .modeling_tf_auto import (
419
        TF_MODEL_MAPPING,
Julien Plu's avatar
Julien Plu committed
420
        TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
421
422
        TF_MODEL_FOR_PRETRAINING_MAPPING,
        TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
Julien Plu's avatar
Julien Plu committed
423
        TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
424
        TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
Julien Plu's avatar
Julien Plu committed
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
        TF_MODEL_WITH_LM_HEAD_MAPPING,
        TFAutoModel,
        TFAutoModelForMultipleChoice,
        TFAutoModelForPreTraining,
        TFAutoModelForQuestionAnswering,
        TFAutoModelForSequenceClassification,
        TFAutoModelForTokenClassification,
        TFAutoModelWithLMHead,
    )

    from .modeling_tf_albert import (
        TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFAlbertForMaskedLM,
        TFAlbertForMultipleChoice,
        TFAlbertForPreTraining,
        TFAlbertForQuestionAnswering,
        TFAlbertForSequenceClassification,
        TFAlbertForTokenClassification,
        TFAlbertMainLayer,
        TFAlbertModel,
        TFAlbertPreTrainedModel,
446
447
448
    )

    from .modeling_tf_bert import (
Julien Plu's avatar
Julien Plu committed
449
        TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
450
451
        TFBertEmbeddings,
        TFBertForMaskedLM,
Julien Plu's avatar
Julien Plu committed
452
        TFBertForMultipleChoice,
453
        TFBertForNextSentencePrediction,
Julien Plu's avatar
Julien Plu committed
454
455
        TFBertForPreTraining,
        TFBertForQuestionAnswering,
456
457
        TFBertForSequenceClassification,
        TFBertForTokenClassification,
Julien Plu's avatar
Julien Plu committed
458
459
460
        TFBertMainLayer,
        TFBertModel,
        TFBertPreTrainedModel,
461
462
    )

Julien Plu's avatar
Julien Plu committed
463
464
465
466
467
468
469
470
    from .modeling_tf_camembert import (
        TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFCamembertForMaskedLM,
        TFCamembertModel,
        TFCamembertForMultipleChoice,
        TFCamembertForQuestionAnswering,
        TFCamembertForSequenceClassification,
        TFCamembertForTokenClassification,
471
472
    )

Julien Plu's avatar
Julien Plu committed
473
474
475
476
477
    from .modeling_tf_ctrl import (
        TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFCTRLLMHeadModel,
        TFCTRLModel,
        TFCTRLPreTrainedModel,
478
479
    )

Julien Plu's avatar
Julien Plu committed
480
481
482
483
484
485
486
487
488
489
    from .modeling_tf_distilbert import (
        TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFDistilBertForMaskedLM,
        TFDistilBertForMultipleChoice,
        TFDistilBertForQuestionAnswering,
        TFDistilBertForSequenceClassification,
        TFDistilBertForTokenClassification,
        TFDistilBertMainLayer,
        TFDistilBertModel,
        TFDistilBertPreTrainedModel,
490
491
    )

Julien Plu's avatar
Julien Plu committed
492
493
494
495
496
497
498
499
    from .modeling_tf_electra import (
        TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFElectraForMaskedLM,
        TFElectraForPreTraining,
        TFElectraForQuestionAnswering,
        TFElectraForTokenClassification,
        TFElectraModel,
        TFElectraPreTrainedModel,
500
501
    )

Julien Plu's avatar
Julien Plu committed
502
503
504
505
506
507
508
509
    from .modeling_tf_flaubert import (
        TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFFlaubertForMultipleChoice,
        TFFlaubertForQuestionAnsweringSimple,
        TFFlaubertForSequenceClassification,
        TFFlaubertForTokenClassification,
        TFFlaubertWithLMHeadModel,
        TFFlaubertModel,
510
511
    )

Julien Plu's avatar
Julien Plu committed
512
513
514
515
516
517
518
519
520
    from .modeling_tf_gpt2 import (
        TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFGPT2DoubleHeadsModel,
        TFGPT2LMHeadModel,
        TFGPT2MainLayer,
        TFGPT2Model,
        TFGPT2PreTrainedModel,
    )

Vasily Shamporov's avatar
Vasily Shamporov committed
521
522
523
524
525
526
527
528
529
530
531
532
533
534
    from .modeling_tf_mobilebert import (
        TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFMobileBertModel,
        TFMobileBertPreTrainedModel,
        TFMobileBertForPreTraining,
        TFMobileBertForSequenceClassification,
        TFMobileBertForQuestionAnswering,
        TFMobileBertForMaskedLM,
        TFMobileBertForNextSentencePrediction,
        TFMobileBertForMultipleChoice,
        TFMobileBertForTokenClassification,
        TFMobileBertMainLayer,
    )

Julien Plu's avatar
Julien Plu committed
535
536
537
538
539
540
541
    from .modeling_tf_openai import (
        TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFOpenAIGPTDoubleHeadsModel,
        TFOpenAIGPTLMHeadModel,
        TFOpenAIGPTMainLayer,
        TFOpenAIGPTModel,
        TFOpenAIGPTPreTrainedModel,
Julien Plu's avatar
Julien Plu committed
542
543
    )

544
    from .modeling_tf_roberta import (
Julien Plu's avatar
Julien Plu committed
545
        TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
546
        TFRobertaForMaskedLM,
Julien Plu's avatar
Julien Plu committed
547
548
        TFRobertaForMultipleChoice,
        TFRobertaForQuestionAnswering,
549
550
        TFRobertaForSequenceClassification,
        TFRobertaForTokenClassification,
Julien Plu's avatar
Julien Plu committed
551
552
553
        TFRobertaMainLayer,
        TFRobertaModel,
        TFRobertaPreTrainedModel,
554
555
    )

Julien Plu's avatar
Julien Plu committed
556
557
558
559
560
    from .modeling_tf_t5 import (
        TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFT5ForConditionalGeneration,
        TFT5Model,
        TFT5PreTrainedModel,
561
562
    )

Julien Plu's avatar
Julien Plu committed
563
564
565
566
567
568
569
    from .modeling_tf_transfo_xl import (
        TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFAdaptiveEmbedding,
        TFTransfoXLLMHeadModel,
        TFTransfoXLMainLayer,
        TFTransfoXLModel,
        TFTransfoXLPreTrainedModel,
570
571
    )

Julien Plu's avatar
Julien Plu committed
572
573
574
575
576
577
578
579
580
581
    from .modeling_tf_xlm import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMWithLMHeadModel,
        TFXLMMainLayer,
        TFXLMModel,
        TFXLMPreTrainedModel,
582
583
    )

Julien Plu's avatar
Julien Plu committed
584
585
586
587
588
589
590
591
    from .modeling_tf_xlm_roberta import (
        TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMRobertaForMaskedLM,
        TFXLMRobertaForMultipleChoice,
        TFXLMRobertaForQuestionAnswering,
        TFXLMRobertaForSequenceClassification,
        TFXLMRobertaForTokenClassification,
        TFXLMRobertaModel,
Julien Plu's avatar
Julien Plu committed
592
    )
593

Julien Plu's avatar
Julien Plu committed
594
595
596
597
598
599
600
601
602
603
    from .modeling_tf_xlnet import (
        TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLNetForMultipleChoice,
        TFXLNetForQuestionAnsweringSimple,
        TFXLNetForSequenceClassification,
        TFXLNetForTokenClassification,
        TFXLNetLMHeadModel,
        TFXLNetMainLayer,
        TFXLNetModel,
        TFXLNetPreTrainedModel,
Lysandre Debut's avatar
Lysandre Debut committed
604
605
    )

606
    # Optimization
Julien Plu's avatar
Julien Plu committed
607
608
609
610
611
612
    from .optimization_tf import (
        AdamWeightDecay,
        create_optimizer,
        GradientAccumulator,
        WarmUp,
    )
Lysandre's avatar
Lysandre committed
613

Julien Plu's avatar
Julien Plu committed
614
615
616
    # Trainer
    from .trainer_tf import TFTrainer

Patrick von Platen's avatar
Patrick von Platen committed
617
    # Benchmarks
618
619
    from .benchmark.benchmark_tf import TensorFlowBenchmark
    from .benchmark.benchmark_args_tf import TensorFlowBenchmarkArguments
Patrick von Platen's avatar
Patrick von Platen committed
620

621

622
if not is_tf_available() and not is_torch_available():
623
624
625
626
627
    logger.warning(
        "Neither PyTorch nor TensorFlow >= 2.0 have been found."
        "Models won't be available and only tokenizers, configuration"
        "and file/data utilities can be used."
    )