Reorganize repo (#8580)

* Put models in subfolders

* Styling

* Fix imports in tests

* More fixes in test imports

* Sneaky hidden imports

* Fix imports in doc files

* More sneaky imports

* Finish fixing tests

* Fix examples

* Fix path for copies

* More fixes for examples

* Fix dummy files

* More fixes for example

* More model import fixes

* Is this why you're unhappy GitHub?

* Fix imports in conver command

tests/test_modeling_tf_distilbert.py

@@ -26,7 +26,7 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers.modeling_tf_distilbert import (
+    from transformers.models.distilbert.modeling_tf_distilbert import (
         TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFDistilBertForMaskedLM,
         TFDistilBertForMultipleChoice,

tests/test_modeling_tf_electra.py

@@ -26,7 +26,7 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers.modeling_tf_electra import (
+    from transformers.models.electra.modeling_tf_electra import (
         TFElectraForMaskedLM,
         TFElectraForMultipleChoice,
         TFElectraForPreTraining,

tests/test_modeling_tf_funnel.py

@@ -26,7 +26,7 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers.modeling_tf_funnel import (
+    from transformers import (
         TFFunnelBaseModel,
         TFFunnelForMaskedLM,
         TFFunnelForMultipleChoice,

tests/test_modeling_tf_gpt2.py

@@ -26,7 +26,7 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers.modeling_tf_gpt2 import (
+    from transformers.models.gpt2.modeling_tf_gpt2 import (
         TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFGPT2DoubleHeadsModel,
         TFGPT2LMHeadModel,

tests/test_modeling_tf_lxmert.py

@@ -25,7 +25,7 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers.modeling_tf_lxmert import TFLxmertForPreTraining, TFLxmertModel
+    from transformers.models.lxmert.modeling_tf_lxmert import TFLxmertForPreTraining, TFLxmertModel
 
 
 class TFLxmertModelTester(object):

tests/test_modeling_tf_mobilebert.py

@@ -26,7 +26,7 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers.modeling_tf_mobilebert import (
+    from transformers import (
         TFMobileBertForMaskedLM,
         TFMobileBertForMultipleChoice,
         TFMobileBertForNextSentencePrediction,

tests/test_modeling_tf_openai.py

@@ -26,7 +26,7 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers.modeling_tf_openai import (
+    from transformers.models.openai.modeling_tf_openai import (
         TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFOpenAIGPTDoubleHeadsModel,
         TFOpenAIGPTLMHeadModel,

tests/test_modeling_tf_pytorch.py

@@ -43,9 +43,9 @@ if is_tf_available():
         TFRobertaForMaskedLM,
         TFT5ForConditionalGeneration,
     )
-    from transformers.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
-    from transformers.modeling_tf_gpt2 import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
-    from transformers.modeling_tf_t5 import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
+    from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
+    from transformers.models.gpt2.modeling_tf_gpt2 import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
+    from transformers.models.t5.modeling_tf_t5 import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
 
 if is_torch_available():
     from transformers import (

tests/test_modeling_tf_roberta.py

@@ -27,7 +27,7 @@ if is_tf_available():
     import numpy
     import tensorflow as tf
 
-    from transformers.modeling_tf_roberta import (
+    from transformers.models.roberta.modeling_tf_roberta import (
         TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFRobertaForMaskedLM,
         TFRobertaForMultipleChoice,

tests/test_modeling_tf_xlnet.py

@@ -27,7 +27,7 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers.modeling_tf_xlnet import (
+    from transformers.models.xlnet.modeling_tf_xlnet import (
         TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFXLNetForMultipleChoice,
         TFXLNetForQuestionAnsweringSimple,

tests/test_modeling_transfo_xl.py

@@ -28,7 +28,7 @@ if is_torch_available():
     import torch
 
     from transformers import TransfoXLConfig, TransfoXLLMHeadModel, TransfoXLModel
-    from transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST
+    from transformers.models.transfo_xl.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST
 
 
 class TransfoXLModelTester:

tests/test_modeling_xlm.py

@@ -37,7 +37,7 @@ if is_torch_available():
         XLMModel,
         XLMWithLMHeadModel,
     )
-    from transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST
+    from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST
 
 
 class XLMModelTester:

tests/test_modeling_xlnet.py

@@ -38,7 +38,7 @@ if is_torch_available():
         XLNetLMHeadModel,
         XLNetModel,
     )
-    from transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_LIST
+    from transformers.models.xlnet.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_LIST
 
 
 class XLNetModelTester:

tests/test_retrieval_rag.py

@@ -10,10 +10,14 @@ import numpy as np
 from datasets import Dataset
 
 from transformers import is_faiss_available
-from transformers.configuration_bart import BartConfig
-from transformers.configuration_dpr import DPRConfig
-from transformers.configuration_rag import RagConfig
-from transformers.retrieval_rag import CustomHFIndex, RagRetriever
+from transformers.models.bart.configuration_bart import BartConfig
+from transformers.models.bart.tokenization_bart import BartTokenizer
+from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
+from transformers.models.dpr.configuration_dpr import DPRConfig
+from transformers.models.dpr.tokenization_dpr import DPRQuestionEncoderTokenizer
+from transformers.models.rag.configuration_rag import RagConfig
+from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
+from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
 from transformers.testing_utils import (
     require_datasets,
     require_faiss,
@@ -21,10 +25,6 @@ from transformers.testing_utils import (
     require_tokenizers,
     require_torch,
 )
-from transformers.tokenization_bart import BartTokenizer
-from transformers.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
-from transformers.tokenization_dpr import DPRQuestionEncoderTokenizer
-from transformers.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
 
 
 if is_faiss_available():
@@ -126,7 +126,7 @@ class RagRetrieverTest(TestCase):
             question_encoder=DPRConfig().to_dict(),
             generator=BartConfig().to_dict(),
         )
-        with patch("transformers.retrieval_rag.load_dataset") as mock_load_dataset:
+        with patch("transformers.models.rag.retrieval_rag.load_dataset") as mock_load_dataset:
             mock_load_dataset.return_value = dataset
             retriever = RagRetriever(
                 config,
@@ -213,7 +213,7 @@ class RagRetrieverTest(TestCase):
     def test_canonical_hf_index_retriever_save_and_from_pretrained(self):
         retriever = self.get_dummy_canonical_hf_index_retriever()
         with tempfile.TemporaryDirectory() as tmp_dirname:
-            with patch("transformers.retrieval_rag.load_dataset") as mock_load_dataset:
+            with patch("transformers.models.rag.retrieval_rag.load_dataset") as mock_load_dataset:
                 mock_load_dataset.return_value = self.get_dummy_dataset()
                 retriever.save_pretrained(tmp_dirname)
                 retriever = RagRetriever.from_pretrained(tmp_dirname)

tests/test_tokenization_auto.py

@@ -27,8 +27,9 @@ from transformers import (
     RobertaTokenizer,
     RobertaTokenizerFast,
 )
-from transformers.configuration_auto import AutoConfig
-from transformers.configuration_roberta import RobertaConfig
+from transformers.models.auto.configuration_auto import AutoConfig
+from transformers.models.auto.tokenization_auto import TOKENIZER_MAPPING
+from transformers.models.roberta.configuration_roberta import RobertaConfig
 from transformers.testing_utils import (
     DUMMY_DIFF_TOKENIZER_IDENTIFIER,
     DUMMY_UNKWOWN_IDENTIFIER,
@@ -36,7 +37,6 @@ from transformers.testing_utils import (
     require_tokenizers,
     slow,
 )
-from transformers.tokenization_auto import TOKENIZER_MAPPING
 
 
 class AutoTokenizerTest(unittest.TestCase):

tests/test_tokenization_bart.py

@@ -4,8 +4,8 @@ import unittest
 
 from transformers import BartTokenizer, BartTokenizerFast, BatchEncoding
 from transformers.file_utils import cached_property
+from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
 from transformers.testing_utils import require_tokenizers, require_torch
-from transformers.tokenization_roberta import VOCAB_FILES_NAMES
 
 from .test_tokenization_common import TokenizerTesterMixin, filter_roberta_detectors
 

tests/test_tokenization_bert.py

@@ -18,8 +18,7 @@ import os
 import unittest
 
 from transformers import BertTokenizerFast
-from transformers.testing_utils import require_tokenizers, slow
-from transformers.tokenization_bert import (
+from transformers.models.bert.tokenization_bert import (
     VOCAB_FILES_NAMES,
     BasicTokenizer,
     BertTokenizer,
@@ -28,6 +27,7 @@ from transformers.tokenization_bert import (
     _is_punctuation,
     _is_whitespace,
 )
+from transformers.testing_utils import require_tokenizers, slow
 
 from .test_tokenization_common import TokenizerTesterMixin, filter_non_english
 

tests/test_tokenization_bert_japanese.py

@@ -18,14 +18,14 @@ import os
 import pickle
 import unittest
 
-from transformers.testing_utils import custom_tokenizers
-from transformers.tokenization_bert_japanese import (
+from transformers.models.bert_japanese.tokenization_bert_japanese import (
     VOCAB_FILES_NAMES,
     BertJapaneseTokenizer,
     CharacterTokenizer,
     MecabTokenizer,
     WordpieceTokenizer,
 )
+from transformers.testing_utils import custom_tokenizers
 
 from .test_tokenization_common import TokenizerTesterMixin
 

tests/test_tokenization_bertweet.py

@@ -16,7 +16,7 @@
 import os
 import unittest
 
-from transformers.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer
+from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer
 
 from .test_tokenization_common import TokenizerTesterMixin
 

tests/test_tokenization_blenderbot.py

@@ -20,7 +20,11 @@ import os
 import unittest
 
 from transformers.file_utils import cached_property
-from transformers.tokenization_blenderbot import VOCAB_FILES_NAMES, BlenderbotSmallTokenizer, BlenderbotTokenizer
+from transformers.models.blenderbot.tokenization_blenderbot import (
+    VOCAB_FILES_NAMES,
+    BlenderbotSmallTokenizer,
+    BlenderbotTokenizer,
+)
 
 from .test_tokenization_common import TokenizerTesterMixin