__init__.py 21.8 KB
Newer Older
1
2
3
4
# flake8: noqa
# There's no way to ignore "F401 '...' imported but unused" warnings in this
# module, but to preserve other warnings. So, don't check this module at all.

Lysandre's avatar
Lysandre committed
5
__version__ = "3.0.2"
thomwolf's avatar
thomwolf committed
6

thomwolf's avatar
thomwolf committed
7
8
9
10
11
12
# Work around to update TensorFlow's absl.logging threshold which alters the
# default Python logging output behavior when present.
# see: https://github.com/abseil/abseil-py/issues/99
# and: https://github.com/tensorflow/tensorflow/issues/26691#issuecomment-500369493
try:
    import absl.logging
13
14
15
except ImportError:
    pass
else:
16
17
    absl.logging.set_verbosity("info")
    absl.logging.set_stderrthreshold("info")
thomwolf's avatar
thomwolf committed
18
    absl.logging._warn_preinit_stderr = False
19

thomwolf's avatar
thomwolf committed
20
21
import logging

Julien Chaumond's avatar
Julien Chaumond committed
22
# Configurations
Aymeric Augustin's avatar
Aymeric Augustin committed
23
from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig
24
from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, CONFIG_MAPPING, AutoConfig
25
from .configuration_bart import BartConfig, MBartConfig
Aymeric Augustin's avatar
Aymeric Augustin committed
26
27
28
29
from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig
from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
from .configuration_distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig
Quentin Lhoest's avatar
Quentin Lhoest committed
30
from .configuration_dpr import DPR_PRETRAINED_CONFIG_ARCHIVE_MAP, DPRConfig
Lysandre Debut's avatar
Lysandre Debut committed
31
from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig
32
from .configuration_encoder_decoder import EncoderDecoderConfig
Hang Le's avatar
Hang Le committed
33
from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig
Aymeric Augustin's avatar
Aymeric Augustin committed
34
from .configuration_gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config
Iz Beltagy's avatar
Iz Beltagy committed
35
from .configuration_longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig
36
from .configuration_marian import MarianConfig
Aymeric Augustin's avatar
Aymeric Augustin committed
37
from .configuration_mmbt import MMBTConfig
Vasily Shamporov's avatar
Vasily Shamporov committed
38
from .configuration_mobilebert import MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileBertConfig
Aymeric Augustin's avatar
Aymeric Augustin committed
39
from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
Patrick von Platen's avatar
Patrick von Platen committed
40
from .configuration_reformer import REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, ReformerConfig
Yacine Jernite's avatar
Yacine Jernite committed
41
from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
Aymeric Augustin's avatar
Aymeric Augustin committed
42
43
44
45
46
47
48
from .configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig
from .configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config
from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
from .configuration_utils import PretrainedConfig
from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig
from .configuration_xlm_roberta import XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMRobertaConfig
from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig
49
from .data import (
Aymeric Augustin's avatar
Aymeric Augustin committed
50
    DataProcessor,
51
52
53
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
Aymeric Augustin's avatar
Aymeric Augustin committed
54
55
56
57
    SquadExample,
    SquadFeatures,
    SquadV1Processor,
    SquadV2Processor,
58
    glue_convert_examples_to_features,
Aymeric Augustin's avatar
Aymeric Augustin committed
59
    glue_output_modes,
60
61
    glue_processors,
    glue_tasks_num_labels,
Aymeric Augustin's avatar
Aymeric Augustin committed
62
63
    is_sklearn_available,
    squad_convert_examples_to_features,
64
65
66
67
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)
68

Aymeric Augustin's avatar
Aymeric Augustin committed
69
70
71
72
73
74
75
76
77
78
79
80
81
# Files and general utilities
from .file_utils import (
    CONFIG_NAME,
    MODEL_CARD_NAME,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    add_end_docstrings,
    add_start_docstrings,
    cached_path,
Patrick von Platen's avatar
Patrick von Platen committed
82
83
84
    is_apex_available,
    is_psutil_available,
    is_py3nvml_available,
Aymeric Augustin's avatar
Aymeric Augustin committed
85
86
    is_tf_available,
    is_torch_available,
87
    is_torch_tpu_available,
Aymeric Augustin's avatar
Aymeric Augustin committed
88
)
89
from .hf_argparser import HfArgumentParser
90

thomwolf's avatar
thomwolf committed
91
# Model Cards
92
from .modelcard import ModelCard
93

Aymeric Augustin's avatar
Aymeric Augustin committed
94
95
96
97
98
99
100
101
102
103
# TF 2.0 <=> PyTorch conversion utilities
from .modeling_tf_pytorch_utils import (
    convert_tf_weight_name_to_pt_weight_name,
    load_pytorch_checkpoint_in_tf2_model,
    load_pytorch_model_in_tf2_model,
    load_pytorch_weights_in_tf2_model,
    load_tf2_checkpoint_in_pytorch_model,
    load_tf2_model_in_pytorch_model,
    load_tf2_weights_in_pytorch_model,
)
104

Aymeric Augustin's avatar
Aymeric Augustin committed
105
106
# Pipelines
from .pipelines import (
107
108
    Conversation,
    ConversationalPipeline,
Aymeric Augustin's avatar
Aymeric Augustin committed
109
110
    CsvPipelineDataFormat,
    FeatureExtractionPipeline,
Julien Chaumond's avatar
Julien Chaumond committed
111
    FillMaskPipeline,
Aymeric Augustin's avatar
Aymeric Augustin committed
112
113
114
115
116
117
    JsonPipelineDataFormat,
    NerPipeline,
    PipedPipelineDataFormat,
    Pipeline,
    PipelineDataFormat,
    QuestionAnsweringPipeline,
118
    SummarizationPipeline,
Aymeric Augustin's avatar
Aymeric Augustin committed
119
    TextClassificationPipeline,
120
    TextGenerationPipeline,
121
    TokenClassificationPipeline,
122
    TranslationPipeline,
Aymeric Augustin's avatar
Aymeric Augustin committed
123
124
    pipeline,
)
Julien Chaumond's avatar
Julien Chaumond committed
125
126

# Tokenizers
Aymeric Augustin's avatar
Aymeric Augustin committed
127
from .tokenization_albert import AlbertTokenizer
128
from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer
Suraj Patil's avatar
Suraj Patil committed
129
from .tokenization_bart import BartTokenizer, BartTokenizerFast, MBartTokenizer
Anthony MOI's avatar
Anthony MOI committed
130
from .tokenization_bert import BasicTokenizer, BertTokenizer, BertTokenizerFast, WordpieceTokenizer
Aymeric Augustin's avatar
Aymeric Augustin committed
131
132
from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer
from .tokenization_camembert import CamembertTokenizer
keskarnitish's avatar
keskarnitish committed
133
from .tokenization_ctrl import CTRLTokenizer
134
from .tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFast
Quentin Lhoest's avatar
Quentin Lhoest committed
135
136
137
138
139
140
141
142
from .tokenization_dpr import (
    DPRContextEncoderTokenizer,
    DPRContextEncoderTokenizerFast,
    DPRQuestionEncoderTokenizer,
    DPRQuestionEncoderTokenizerFast,
    DPRReaderTokenizer,
    DPRReaderTokenizerFast,
)
Lysandre Debut's avatar
Lysandre Debut committed
143
from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast
Hang Le's avatar
Hang Le committed
144
from .tokenization_flaubert import FlaubertTokenizer
Anthony MOI's avatar
Anthony MOI committed
145
from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
Suraj Patil's avatar
Suraj Patil committed
146
from .tokenization_longformer import LongformerTokenizer, LongformerTokenizerFast
Vasily Shamporov's avatar
Vasily Shamporov committed
147
from .tokenization_mobilebert import MobileBertTokenizer, MobileBertTokenizerFast
148
from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
Patrick von Platen's avatar
Patrick von Platen committed
149
from .tokenization_reformer import ReformerTokenizer
Yacine Jernite's avatar
Yacine Jernite committed
150
from .tokenization_retribert import RetriBertTokenizer, RetriBertTokenizerFast
151
from .tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
thomwolf's avatar
thomwolf committed
152
from .tokenization_t5 import T5Tokenizer
153
from .tokenization_transfo_xl import TransfoXLCorpus, TransfoXLTokenizer, TransfoXLTokenizerFast
154
155
from .tokenization_utils import PreTrainedTokenizer
from .tokenization_utils_base import (
156
    BatchEncoding,
157
158
    CharSpan,
    PreTrainedTokenizerBase,
159
160
    SpecialTokensMixin,
    TensorType,
161
    TokenSpan,
162
)
163
from .tokenization_utils_fast import PreTrainedTokenizerFast
Aymeric Augustin's avatar
Aymeric Augustin committed
164
from .tokenization_xlm import XLMTokenizer
165
from .tokenization_xlm_roberta import XLMRobertaTokenizer
Aymeric Augustin's avatar
Aymeric Augustin committed
166
from .tokenization_xlnet import SPIECE_UNDERLINE, XLNetTokenizer
167
168

# Trainer
169
from .trainer_utils import EvalPrediction, set_seed
170
from .training_args import TrainingArguments
Julien Plu's avatar
Julien Plu committed
171
from .training_args_tf import TFTrainingArguments
Aymeric Augustin's avatar
Aymeric Augustin committed
172
173
174
175
176
177
178


logger = logging.getLogger(__name__)  # pylint: disable=invalid-name


if is_sklearn_available():
    from .data import glue_compute_metrics, xnli_compute_metrics
179
180


181
# Modeling
thomwolf's avatar
thomwolf committed
182
if is_torch_available():
183
184
    from .generation_utils import top_k_top_p_filtering
    from .modeling_utils import PreTrainedModel, prune_layer, Conv1D, apply_chunking_to_forward
185
186
    from .modeling_auto import (
        AutoModel,
thomwolf's avatar
thomwolf committed
187
        AutoModelForPreTraining,
188
189
190
        AutoModelForSequenceClassification,
        AutoModelForQuestionAnswering,
        AutoModelWithLMHead,
191
192
193
        AutoModelForCausalLM,
        AutoModelForMaskedLM,
        AutoModelForSeq2SeqLM,
194
        AutoModelForTokenClassification,
Julien Chaumond's avatar
Julien Chaumond committed
195
        AutoModelForMultipleChoice,
196
197
198
        MODEL_MAPPING,
        MODEL_FOR_PRETRAINING_MAPPING,
        MODEL_WITH_LM_HEAD_MAPPING,
199
200
201
        MODEL_FOR_CAUSAL_LM_MAPPING,
        MODEL_FOR_MASKED_LM_MAPPING,
        MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
202
203
204
        MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
        MODEL_FOR_QUESTION_ANSWERING_MAPPING,
        MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
Julien Chaumond's avatar
Julien Chaumond committed
205
        MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
206
207
    )

Vasily Shamporov's avatar
Vasily Shamporov committed
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
    from .modeling_mobilebert import (
        MobileBertPreTrainedModel,
        MobileBertModel,
        MobileBertForPreTraining,
        MobileBertForSequenceClassification,
        MobileBertForQuestionAnswering,
        MobileBertForMaskedLM,
        MobileBertForNextSentencePrediction,
        MobileBertForMultipleChoice,
        MobileBertForTokenClassification,
        load_tf_weights_in_mobilebert,
        MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        MobileBertLayer,
    )

223
224
225
226
227
    from .modeling_bert import (
        BertPreTrainedModel,
        BertModel,
        BertForPreTraining,
        BertForMaskedLM,
228
        BertLMHeadModel,
229
230
231
232
233
234
        BertForNextSentencePrediction,
        BertForSequenceClassification,
        BertForMultipleChoice,
        BertForTokenClassification,
        BertForQuestionAnswering,
        load_tf_weights_in_bert,
235
        BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
Patrick von Platen's avatar
Patrick von Platen committed
236
        BertLayer,
237
238
239
240
241
242
243
    )
    from .modeling_openai import (
        OpenAIGPTPreTrainedModel,
        OpenAIGPTModel,
        OpenAIGPTLMHeadModel,
        OpenAIGPTDoubleHeadsModel,
        load_tf_weights_in_openai_gpt,
244
        OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
245
246
247
248
249
250
251
    )
    from .modeling_transfo_xl import (
        TransfoXLPreTrainedModel,
        TransfoXLModel,
        TransfoXLLMHeadModel,
        AdaptiveEmbedding,
        load_tf_weights_in_transfo_xl,
252
        TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
253
254
255
256
257
258
259
    )
    from .modeling_gpt2 import (
        GPT2PreTrainedModel,
        GPT2Model,
        GPT2LMHeadModel,
        GPT2DoubleHeadsModel,
        load_tf_weights_in_gpt2,
260
        GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
261
    )
262
    from .modeling_ctrl import CTRLPreTrainedModel, CTRLModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_LIST
263
264
265
266
267
268
269
270
271
272
    from .modeling_xlnet import (
        XLNetPreTrainedModel,
        XLNetModel,
        XLNetLMHeadModel,
        XLNetForSequenceClassification,
        XLNetForTokenClassification,
        XLNetForMultipleChoice,
        XLNetForQuestionAnsweringSimple,
        XLNetForQuestionAnswering,
        load_tf_weights_in_xlnet,
273
        XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
274
275
276
277
278
279
    )
    from .modeling_xlm import (
        XLMPreTrainedModel,
        XLMModel,
        XLMWithLMHeadModel,
        XLMForSequenceClassification,
280
        XLMForTokenClassification,
281
282
        XLMForQuestionAnswering,
        XLMForQuestionAnsweringSimple,
283
        XLMForMultipleChoice,
284
        XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
285
    )
286
    from .modeling_bart import (
287
        PretrainedBartModel,
288
289
290
        BartForSequenceClassification,
        BartModel,
        BartForConditionalGeneration,
Suraj Patil's avatar
Suraj Patil committed
291
        BartForQuestionAnswering,
292
        BART_PRETRAINED_MODEL_ARCHIVE_LIST,
293
    )
294
    from .modeling_marian import MarianMTModel
295
    from .tokenization_marian import MarianTokenizer
296
297
298
299
300
301
302
    from .modeling_roberta import (
        RobertaForMaskedLM,
        RobertaModel,
        RobertaForSequenceClassification,
        RobertaForMultipleChoice,
        RobertaForTokenClassification,
        RobertaForQuestionAnswering,
303
        ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
304
305
306
307
308
    )
    from .modeling_distilbert import (
        DistilBertPreTrainedModel,
        DistilBertForMaskedLM,
        DistilBertModel,
309
        DistilBertForMultipleChoice,
310
311
312
        DistilBertForSequenceClassification,
        DistilBertForQuestionAnswering,
        DistilBertForTokenClassification,
313
        DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
314
315
316
317
318
319
320
    )
    from .modeling_camembert import (
        CamembertForMaskedLM,
        CamembertModel,
        CamembertForSequenceClassification,
        CamembertForMultipleChoice,
        CamembertForTokenClassification,
321
        CamembertForQuestionAnswering,
322
        CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
323
    )
324
    from .modeling_encoder_decoder import EncoderDecoderModel
325
326
327
    from .modeling_t5 import (
        T5PreTrainedModel,
        T5Model,
328
        T5ForConditionalGeneration,
329
        load_tf_weights_in_t5,
330
        T5_PRETRAINED_MODEL_ARCHIVE_LIST,
331
332
333
334
    )
    from .modeling_albert import (
        AlbertPreTrainedModel,
        AlbertModel,
335
        AlbertForPreTraining,
336
        AlbertForMaskedLM,
337
        AlbertForMultipleChoice,
338
339
        AlbertForSequenceClassification,
        AlbertForQuestionAnswering,
340
        AlbertForTokenClassification,
341
        load_tf_weights_in_albert,
342
        ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
343
344
345
346
347
348
349
    )
    from .modeling_xlm_roberta import (
        XLMRobertaForMaskedLM,
        XLMRobertaModel,
        XLMRobertaForMultipleChoice,
        XLMRobertaForSequenceClassification,
        XLMRobertaForTokenClassification,
350
        XLMRobertaForQuestionAnswering,
351
        XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
352
    )
353
354
    from .modeling_mmbt import ModalEmbeddings, MMBTModel, MMBTForClassification

Hang Le's avatar
Hang Le committed
355
356
357
358
    from .modeling_flaubert import (
        FlaubertModel,
        FlaubertWithLMHeadModel,
        FlaubertForSequenceClassification,
359
        FlaubertForTokenClassification,
Hang Le's avatar
Hang Le committed
360
361
        FlaubertForQuestionAnswering,
        FlaubertForQuestionAnsweringSimple,
362
363
        FlaubertForTokenClassification,
        FlaubertForMultipleChoice,
364
        FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
Hang Le's avatar
Hang Le committed
365
366
    )

Lysandre Debut's avatar
Lysandre Debut committed
367
368
369
370
    from .modeling_electra import (
        ElectraForPreTraining,
        ElectraForMaskedLM,
        ElectraForTokenClassification,
371
        ElectraPreTrainedModel,
Suraj Patil's avatar
Suraj Patil committed
372
        ElectraForMultipleChoice,
373
        ElectraForSequenceClassification,
374
        ElectraForQuestionAnswering,
Lysandre Debut's avatar
Lysandre Debut committed
375
376
        ElectraModel,
        load_tf_weights_in_electra,
377
        ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
Lysandre Debut's avatar
Lysandre Debut committed
378
379
    )

Patrick von Platen's avatar
Patrick von Platen committed
380
381
382
383
    from .modeling_reformer import (
        ReformerAttention,
        ReformerLayer,
        ReformerModel,
384
        ReformerForMaskedLM,
Patrick von Platen's avatar
Patrick von Platen committed
385
        ReformerModelWithLMHead,
386
        ReformerForSequenceClassification,
387
        ReformerForQuestionAnswering,
388
        REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
Patrick von Platen's avatar
Patrick von Platen committed
389
390
    )

391
392
393
    from .modeling_longformer import (
        LongformerModel,
        LongformerForMaskedLM,
394
        LongformerForSequenceClassification,
395
        LongformerForMultipleChoice,
396
        LongformerForTokenClassification,
397
        LongformerForQuestionAnswering,
398
        LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
399
    )
Iz Beltagy's avatar
Iz Beltagy committed
400

Quentin Lhoest's avatar
Quentin Lhoest committed
401
402
403
404
405
406
407
408
    from .modeling_dpr import (
        DPRPretrainedContextEncoder,
        DPRPretrainedQuestionEncoder,
        DPRPretrainedReader,
        DPRContextEncoder,
        DPRQuestionEncoder,
        DPRReader,
    )
Yacine Jernite's avatar
Yacine Jernite committed
409
410
411
412
413
414
    from .modeling_retribert import (
        RetriBertPreTrainedModel,
        RetriBertModel,
        RETRIBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
    )

thomwolf's avatar
thomwolf committed
415
    # Optimization
416
417
418
419
420
421
422
423
    from .optimization import (
        AdamW,
        get_constant_schedule,
        get_constant_schedule_with_warmup,
        get_cosine_schedule_with_warmup,
        get_cosine_with_hard_restarts_schedule_with_warmup,
        get_linear_schedule_with_warmup,
    )
thomwolf's avatar
thomwolf committed
424

Julien Chaumond's avatar
Julien Chaumond committed
425
    # Trainer
426
    from .trainer import Trainer, set_seed, torch_distributed_zero_first, EvalPrediction
427
428
429
430
431
432
    from .data.data_collator import (
        default_data_collator,
        DataCollator,
        DataCollatorForLanguageModeling,
        DataCollatorForPermutationLanguageModeling,
    )
433
434
435
436
437
438
439
440
    from .data.datasets import (
        GlueDataset,
        TextDataset,
        LineByLineTextDataset,
        GlueDataTrainingArguments,
        SquadDataset,
        SquadDataTrainingArguments,
    )
thomwolf's avatar
thomwolf committed
441

442
    # Benchmarks
Patrick von Platen's avatar
Patrick von Platen committed
443
444
    from .benchmark.benchmark import PyTorchBenchmark
    from .benchmark.benchmark_args import PyTorchBenchmarkArguments
445

thomwolf's avatar
thomwolf committed
446
# TensorFlow
thomwolf's avatar
thomwolf committed
447
if is_tf_available():
448
    from .generation_tf_utils import tf_top_k_top_p_filtering
449
450
    from .modeling_tf_utils import (
        shape_list,
Julien Plu's avatar
Julien Plu committed
451
452
453
        TFPreTrainedModel,
        TFSequenceSummary,
        TFSharedEmbeddings,
454
    )
455
    from .modeling_tf_auto import (
456
        TF_MODEL_MAPPING,
Julien Plu's avatar
Julien Plu committed
457
        TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
458
459
        TF_MODEL_FOR_PRETRAINING_MAPPING,
        TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
Julien Plu's avatar
Julien Plu committed
460
        TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
461
        TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
Julien Plu's avatar
Julien Plu committed
462
        TF_MODEL_WITH_LM_HEAD_MAPPING,
463
464
465
        TF_MODEL_FOR_CAUSAL_LM_MAPPING,
        TF_MODEL_FOR_MASKED_LM_MAPPING,
        TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
Julien Plu's avatar
Julien Plu committed
466
467
468
469
470
471
472
        TFAutoModel,
        TFAutoModelForMultipleChoice,
        TFAutoModelForPreTraining,
        TFAutoModelForQuestionAnswering,
        TFAutoModelForSequenceClassification,
        TFAutoModelForTokenClassification,
        TFAutoModelWithLMHead,
473
474
475
        TFAutoModelForCausalLM,
        TFAutoModelForMaskedLM,
        TFAutoModelForSeq2SeqLM,
Julien Plu's avatar
Julien Plu committed
476
477
478
479
480
481
482
483
484
485
486
487
488
    )

    from .modeling_tf_albert import (
        TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFAlbertForMaskedLM,
        TFAlbertForMultipleChoice,
        TFAlbertForPreTraining,
        TFAlbertForQuestionAnswering,
        TFAlbertForSequenceClassification,
        TFAlbertForTokenClassification,
        TFAlbertMainLayer,
        TFAlbertModel,
        TFAlbertPreTrainedModel,
489
490
491
    )

    from .modeling_tf_bert import (
Julien Plu's avatar
Julien Plu committed
492
        TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
493
        TFBertEmbeddings,
494
        TFBertLMHeadModel,
495
        TFBertForMaskedLM,
Julien Plu's avatar
Julien Plu committed
496
        TFBertForMultipleChoice,
497
        TFBertForNextSentencePrediction,
Julien Plu's avatar
Julien Plu committed
498
499
        TFBertForPreTraining,
        TFBertForQuestionAnswering,
500
501
        TFBertForSequenceClassification,
        TFBertForTokenClassification,
Julien Plu's avatar
Julien Plu committed
502
503
504
        TFBertMainLayer,
        TFBertModel,
        TFBertPreTrainedModel,
505
506
    )

Julien Plu's avatar
Julien Plu committed
507
508
509
510
511
512
513
514
    from .modeling_tf_camembert import (
        TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFCamembertForMaskedLM,
        TFCamembertModel,
        TFCamembertForMultipleChoice,
        TFCamembertForQuestionAnswering,
        TFCamembertForSequenceClassification,
        TFCamembertForTokenClassification,
515
516
    )

Julien Plu's avatar
Julien Plu committed
517
518
519
520
521
    from .modeling_tf_ctrl import (
        TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFCTRLLMHeadModel,
        TFCTRLModel,
        TFCTRLPreTrainedModel,
522
523
    )

Julien Plu's avatar
Julien Plu committed
524
525
526
527
528
529
530
531
532
533
    from .modeling_tf_distilbert import (
        TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFDistilBertForMaskedLM,
        TFDistilBertForMultipleChoice,
        TFDistilBertForQuestionAnswering,
        TFDistilBertForSequenceClassification,
        TFDistilBertForTokenClassification,
        TFDistilBertMainLayer,
        TFDistilBertModel,
        TFDistilBertPreTrainedModel,
534
535
    )

Julien Plu's avatar
Julien Plu committed
536
537
538
539
540
541
542
543
    from .modeling_tf_electra import (
        TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFElectraForMaskedLM,
        TFElectraForPreTraining,
        TFElectraForQuestionAnswering,
        TFElectraForTokenClassification,
        TFElectraModel,
        TFElectraPreTrainedModel,
544
545
    )

Julien Plu's avatar
Julien Plu committed
546
547
548
549
550
551
552
553
    from .modeling_tf_flaubert import (
        TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFFlaubertForMultipleChoice,
        TFFlaubertForQuestionAnsweringSimple,
        TFFlaubertForSequenceClassification,
        TFFlaubertForTokenClassification,
        TFFlaubertWithLMHeadModel,
        TFFlaubertModel,
554
555
    )

Julien Plu's avatar
Julien Plu committed
556
557
558
559
560
561
562
563
564
    from .modeling_tf_gpt2 import (
        TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFGPT2DoubleHeadsModel,
        TFGPT2LMHeadModel,
        TFGPT2MainLayer,
        TFGPT2Model,
        TFGPT2PreTrainedModel,
    )

Vasily Shamporov's avatar
Vasily Shamporov committed
565
566
567
568
569
570
571
572
573
574
575
576
577
578
    from .modeling_tf_mobilebert import (
        TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFMobileBertModel,
        TFMobileBertPreTrainedModel,
        TFMobileBertForPreTraining,
        TFMobileBertForSequenceClassification,
        TFMobileBertForQuestionAnswering,
        TFMobileBertForMaskedLM,
        TFMobileBertForNextSentencePrediction,
        TFMobileBertForMultipleChoice,
        TFMobileBertForTokenClassification,
        TFMobileBertMainLayer,
    )

Julien Plu's avatar
Julien Plu committed
579
580
581
582
583
584
585
    from .modeling_tf_openai import (
        TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFOpenAIGPTDoubleHeadsModel,
        TFOpenAIGPTLMHeadModel,
        TFOpenAIGPTMainLayer,
        TFOpenAIGPTModel,
        TFOpenAIGPTPreTrainedModel,
Julien Plu's avatar
Julien Plu committed
586
587
    )

588
    from .modeling_tf_roberta import (
Julien Plu's avatar
Julien Plu committed
589
        TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
590
        TFRobertaForMaskedLM,
Julien Plu's avatar
Julien Plu committed
591
592
        TFRobertaForMultipleChoice,
        TFRobertaForQuestionAnswering,
593
594
        TFRobertaForSequenceClassification,
        TFRobertaForTokenClassification,
Julien Plu's avatar
Julien Plu committed
595
596
597
        TFRobertaMainLayer,
        TFRobertaModel,
        TFRobertaPreTrainedModel,
598
599
    )

Julien Plu's avatar
Julien Plu committed
600
601
602
603
604
    from .modeling_tf_t5 import (
        TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFT5ForConditionalGeneration,
        TFT5Model,
        TFT5PreTrainedModel,
605
606
    )

Julien Plu's avatar
Julien Plu committed
607
608
609
610
611
612
613
    from .modeling_tf_transfo_xl import (
        TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFAdaptiveEmbedding,
        TFTransfoXLLMHeadModel,
        TFTransfoXLMainLayer,
        TFTransfoXLModel,
        TFTransfoXLPreTrainedModel,
614
615
    )

Julien Plu's avatar
Julien Plu committed
616
617
618
619
620
621
622
623
624
625
    from .modeling_tf_xlm import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMWithLMHeadModel,
        TFXLMMainLayer,
        TFXLMModel,
        TFXLMPreTrainedModel,
626
627
    )

Julien Plu's avatar
Julien Plu committed
628
629
630
631
632
633
634
635
    from .modeling_tf_xlm_roberta import (
        TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMRobertaForMaskedLM,
        TFXLMRobertaForMultipleChoice,
        TFXLMRobertaForQuestionAnswering,
        TFXLMRobertaForSequenceClassification,
        TFXLMRobertaForTokenClassification,
        TFXLMRobertaModel,
Julien Plu's avatar
Julien Plu committed
636
    )
637

Julien Plu's avatar
Julien Plu committed
638
639
640
641
642
643
644
645
646
647
    from .modeling_tf_xlnet import (
        TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLNetForMultipleChoice,
        TFXLNetForQuestionAnsweringSimple,
        TFXLNetForSequenceClassification,
        TFXLNetForTokenClassification,
        TFXLNetLMHeadModel,
        TFXLNetMainLayer,
        TFXLNetModel,
        TFXLNetPreTrainedModel,
Lysandre Debut's avatar
Lysandre Debut committed
648
649
    )

650
    # Optimization
Julien Plu's avatar
Julien Plu committed
651
652
653
654
655
656
    from .optimization_tf import (
        AdamWeightDecay,
        create_optimizer,
        GradientAccumulator,
        WarmUp,
    )
Lysandre's avatar
Lysandre committed
657

Julien Plu's avatar
Julien Plu committed
658
659
660
    # Trainer
    from .trainer_tf import TFTrainer

Patrick von Platen's avatar
Patrick von Platen committed
661
    # Benchmarks
662
663
    from .benchmark.benchmark_tf import TensorFlowBenchmark
    from .benchmark.benchmark_args_tf import TensorFlowBenchmarkArguments
Patrick von Platen's avatar
Patrick von Platen committed
664

665

666
if not is_tf_available() and not is_torch_available():
667
668
669
670
671
    logger.warning(
        "Neither PyTorch nor TensorFlow >= 2.0 have been found."
        "Models won't be available and only tokenizers, configuration"
        "and file/data utilities can be used."
    )