Reorganize repo (#8580)

* Put models in subfolders

* Styling

* Fix imports in tests

* More fixes in test imports

* Sneaky hidden imports

* Fix imports in doc files

* More sneaky imports

* Finish fixing tests

* Fix examples

* Fix path for copies

* More fixes for examples

* Fix dummy files

* More fixes for example

* More model import fixes

* Is this why you're unhappy GitHub?

* Fix imports in conver command

model_cards/deepset/minilm-uncased-squad2/README.md

@@ -48,9 +48,7 @@ Evaluated on the SQuAD 2.0 dev set with the [official eval script](https://works
 
 ### In Transformers
 ```python
-from transformers.pipelines import pipeline
-from transformers.modeling_auto import AutoModelForQuestionAnswering
-from transformers.tokenization_auto import AutoTokenizer
+from transformers import AutoModelForQuestionAnswering,  AutoTokenizer, pipeline
 
 model_name = "deepset/minilm-uncased-squad2"
 

model_cards/deepset/roberta-base-squad2-covid/README.md

@@ -39,9 +39,8 @@ This model is the model obtained from the **third** fold of the cross-validation
 
 ### In Transformers
 ```python
-from transformers.pipelines import pipeline
-from transformers.modeling_auto import AutoModelForQuestionAnswering
-from transformers.tokenization_auto import AutoTokenizer
+from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline
+
 
 model_name = "deepset/roberta-base-squad2-covid"
 

model_cards/deepset/roberta-base-squad2-v2/README.md

@@ -48,9 +48,7 @@ Evaluated on the SQuAD 2.0 dev set with the [official eval script](https://works
 
 ### In Transformers
 ```python
-from transformers.pipelines import pipeline
-from transformers.modeling_auto import AutoModelForQuestionAnswering
-from transformers.tokenization_auto import AutoTokenizer
+from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline
 
 model_name = "deepset/roberta-base-squad2-v2"
 

model_cards/deepset/roberta-base-squad2/README.md

@@ -54,9 +54,7 @@ Evaluated on the SQuAD 2.0 dev set with the [official eval script](https://works
 
 ### In Transformers
 ```python
-from transformers.pipelines import pipeline
-from transformers.modeling_auto import AutoModelForQuestionAnswering
-from transformers.tokenization_auto import AutoTokenizer
+from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline
 
 model_name = "deepset/roberta-base-squad2"
 

model_cards/deepset/xlm-roberta-large-squad2/README.md

@@ -63,9 +63,7 @@ Evaluated on German [XQuAD: xquad.de.json](https://github.com/deepmind/xquad)
 
 ### In Transformers
 ```python
-from transformers.pipelines import pipeline
-from transformers.modeling_auto import AutoModelForQuestionAnswering
-from transformers.tokenization_auto import AutoTokenizer
+from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline
 
 model_name = "deepset/xlm-roberta-large-squad2"
 

model_cards/facebook/wmt19-de-en/README.md

@@ -36,8 +36,7 @@ All four models are available:
 #### How to use
 
 ```python
-from transformers.tokenization_fsmt import FSMTTokenizer
-from transformers.modeling_fsmt import FSMTForConditionalGeneration
+from transformers import FSMTForConditionalGeneration, FSMTTokenizer
 mname = "facebook/wmt19-de-en"
 tokenizer = FSMTTokenizer.from_pretrained(mname)
 model = FSMTForConditionalGeneration.from_pretrained(mname)

model_cards/facebook/wmt19-en-de/README.md

@@ -36,8 +36,7 @@ All four models are available:
 #### How to use
 
 ```python
-from transformers.tokenization_fsmt import FSMTTokenizer
-from transformers.modeling_fsmt import FSMTForConditionalGeneration
+from transformers import FSMTForConditionalGeneration, FSMTTokenizer
 mname = "facebook/wmt19-en-de"
 tokenizer = FSMTTokenizer.from_pretrained(mname)
 model = FSMTForConditionalGeneration.from_pretrained(mname)

model_cards/facebook/wmt19-en-ru/README.md

@@ -36,8 +36,7 @@ All four models are available:
 #### How to use
 
 ```python
-from transformers.tokenization_fsmt import FSMTTokenizer
-from transformers.modeling_fsmt import FSMTForConditionalGeneration
+from transformers import FSMTForConditionalGeneration, FSMTTokenizer
 mname = "facebook/wmt19-en-ru"
 tokenizer = FSMTTokenizer.from_pretrained(mname)
 model = FSMTForConditionalGeneration.from_pretrained(mname)

model_cards/facebook/wmt19-ru-en/README.md

@@ -36,8 +36,7 @@ All four models are available:
 #### How to use
 
 ```python
-from transformers.tokenization_fsmt import FSMTTokenizer
-from transformers.modeling_fsmt import FSMTForConditionalGeneration
+from transformers import FSMTForConditionalGeneration, FSMTTokenizer
 mname = "facebook/wmt19-ru-en"
 tokenizer = FSMTTokenizer.from_pretrained(mname)
 model = FSMTForConditionalGeneration.from_pretrained(mname)

model_cards/julien-c/bert-xsmall-dummy/README.md

@@ -2,11 +2,7 @@
 
 
 ```python
-from transformers.configuration_bert import BertConfig
-from transformers.modeling_bert import BertForMaskedLM
-from transformers.modeling_tf_bert import TFBertForMaskedLM
-from transformers.tokenization_bert import BertTokenizer
-
+from transformers BertConfig, BertForMaskedLM, BertTokenizer, TFBertForMaskedLM
 
 SMALL_MODEL_IDENTIFIER = "julien-c/bert-xsmall-dummy"
 DIRNAME = "./bert-xsmall-dummy"

scripts/fsmt/gen-card-allenai-wmt16.py

@@ -60,8 +60,7 @@ All 3 models are available:
 #### How to use
 
 ```python
-from transformers.tokenization_fsmt import FSMTTokenizer
-from transformers.modeling_fsmt import FSMTForConditionalGeneration
+from transformers import FSMTForConditionalGeneration, FSMTTokenizer
 mname = "allenai/{model_name}"
 tokenizer = FSMTTokenizer.from_pretrained(mname)
 model = FSMTForConditionalGeneration.from_pretrained(mname)

scripts/fsmt/gen-card-allenai-wmt19.py

@@ -59,8 +59,7 @@ For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the S
 #### How to use
 
 ```python
-from transformers.tokenization_fsmt import FSMTTokenizer
-from transformers.modeling_fsmt import FSMTForConditionalGeneration
+from transformers import FSMTForConditionalGeneration, FSMTTokenizer
 mname = "allenai/{model_name}"
 tokenizer = FSMTTokenizer.from_pretrained(mname)
 model = FSMTForConditionalGeneration.from_pretrained(mname)

scripts/fsmt/gen-card-facebook-wmt19.py

@@ -63,8 +63,7 @@ All four models are available:
 #### How to use
 
 ```python
-from transformers.tokenization_fsmt import FSMTTokenizer
-from transformers.modeling_fsmt import FSMTForConditionalGeneration
+from transformers import FSMTForConditionalGeneration, FSMTTokenizer
 mname = "facebook/wmt19-{src_lang}-{tgt_lang}"
 tokenizer = FSMTTokenizer.from_pretrained(mname)
 model = FSMTForConditionalGeneration.from_pretrained(mname)

src/transformers/benchmark/benchmark.py

@@ -23,7 +23,7 @@ from typing import Callable, Optional
 
 from ..configuration_utils import PretrainedConfig
 from ..file_utils import is_py3nvml_available, is_torch_available
-from ..modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
+from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
 from ..utils import logging
 from .benchmark_utils import (
     Benchmark,

src/transformers/benchmark/benchmark_tf.py

@@ -25,7 +25,7 @@ from typing import Callable, Optional
 
 from ..configuration_utils import PretrainedConfig
 from ..file_utils import is_py3nvml_available, is_tf_available
-from ..modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
+from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
 from ..utils import logging
 from .benchmark_utils import (
     Benchmark,

src/transformers/benchmark/benchmark_utils.py

@@ -327,7 +327,7 @@ def start_memory_tracing(
 
         - `modules_to_trace`: (None, string, list/tuple of string) if None, all events are recorded if string or list
           of strings: only events from the listed module/sub-module will be recorded (e.g. 'fairseq' or
-          'transformers.modeling_gpt2')
+          'transformers.models.gpt2.modeling_gpt2')
         - `modules_not_to_trace`: (None, string, list/tuple of string) if None, no module is avoided if string or list
           of strings: events from the listed module/sub-module will not be recorded (e.g. 'torch')
         - `events_to_trace`: string or list of string of events to be recorded (see official python doc for

src/transformers/commands/convert.py

@@ -73,7 +73,7 @@ class ConvertCommand(BaseTransformersCLICommand):
     def run(self):
         if self._model_type == "albert":
             try:
-                from transformers.convert_albert_original_tf_checkpoint_to_pytorch import (
+                from transformers.models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
                     convert_tf_checkpoint_to_pytorch,
                 )
             except ImportError:
@@ -82,7 +82,7 @@ class ConvertCommand(BaseTransformersCLICommand):
             convert_tf_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
         elif self._model_type == "bert":
             try:
-                from transformers.convert_bert_original_tf_checkpoint_to_pytorch import (
+                from transformers.models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
                     convert_tf_checkpoint_to_pytorch,
                 )
             except ImportError:
@@ -91,7 +91,7 @@ class ConvertCommand(BaseTransformersCLICommand):
             convert_tf_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
         elif self._model_type == "funnel":
             try:
-                from transformers.convert_funnel_original_tf_checkpoint_to_pytorch import (
+                from transformers.models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
                     convert_tf_checkpoint_to_pytorch,
                 )
             except ImportError:
@@ -99,14 +99,14 @@ class ConvertCommand(BaseTransformersCLICommand):
 
             convert_tf_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
         elif self._model_type == "gpt":
-            from transformers.convert_openai_original_tf_checkpoint_to_pytorch import (
+            from transformers.models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
                 convert_openai_checkpoint_to_pytorch,
             )
 
             convert_openai_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
         elif self._model_type == "transfo_xl":
             try:
-                from transformers.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
+                from transformers.models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
                     convert_transfo_xl_checkpoint_to_pytorch,
                 )
             except ImportError:
@@ -123,7 +123,7 @@ class ConvertCommand(BaseTransformersCLICommand):
             )
         elif self._model_type == "gpt2":
             try:
-                from transformers.convert_gpt2_original_tf_checkpoint_to_pytorch import (
+                from transformers.models.gpt2.convert_gpt2_original_tf_checkpoint_to_pytorch import (
                     convert_gpt2_checkpoint_to_pytorch,
                 )
             except ImportError:
@@ -132,7 +132,7 @@ class ConvertCommand(BaseTransformersCLICommand):
             convert_gpt2_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
         elif self._model_type == "xlnet":
             try:
-                from transformers.convert_xlnet_original_tf_checkpoint_to_pytorch import (
+                from transformers.models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
                     convert_xlnet_checkpoint_to_pytorch,
                 )
             except ImportError:
@@ -142,13 +142,13 @@ class ConvertCommand(BaseTransformersCLICommand):
                 self._tf_checkpoint, self._config, self._pytorch_dump_output, self._finetuning_task_name
             )
         elif self._model_type == "xlm":
-            from transformers.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
+            from transformers.models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
                 convert_xlm_checkpoint_to_pytorch,
             )
 
             convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint, self._pytorch_dump_output)
         elif self._model_type == "lxmert":
-            from transformers.convert_lxmert_original_pytorch_checkpoint_to_pytorch import (
+            from transformers.models.lxmert.convert_lxmert_original_pytorch_checkpoint_to_pytorch import (
                 convert_lxmert_checkpoint_to_pytorch,
             )
 

src/transformers/data/datasets/squad.py

@@ -9,7 +9,7 @@ from torch.utils.data.dataset import Dataset
 
 from filelock import FileLock
 
-from ...modeling_auto import MODEL_FOR_QUESTION_ANSWERING_MAPPING
+from ...models.auto.modeling_auto import MODEL_FOR_QUESTION_ANSWERING_MAPPING
 from ...tokenization_utils import PreTrainedTokenizer
 from ...utils import logging
 from ..processors.squad import SquadFeatures, SquadV1Processor, SquadV2Processor, squad_convert_examples_to_features

src/transformers/data/metrics/squad_metrics.py

@@ -14,7 +14,7 @@ import math
 import re
 import string
 
-from transformers.tokenization_bert import BasicTokenizer
+from transformers import BasicTokenizer
 
 from ...utils import logging