Reorganize repo (#8580)

* Put models in subfolders

* Styling

* Fix imports in tests

* More fixes in test imports

* Sneaky hidden imports

* Fix imports in doc files

* More sneaky imports

* Finish fixing tests

* Fix examples

* Fix path for copies

* More fixes for examples

* Fix dummy files

* More fixes for example

* More model import fixes

* Is this why you're unhappy GitHub?

* Fix imports in conver command

src/transformers/modeling_tf_electra.py → src/transformers/models/electra/modeling_tf_electra.py

@@ -4,9 +4,8 @@ from typing import Optional, Tuple
 
 import tensorflow as tf
 
-from .activations_tf import get_tf_activation
-from .configuration_electra import ElectraConfig
-from .file_utils import (
+from ...activations_tf import get_tf_activation
+from ...file_utils import (
     MULTIPLE_CHOICE_DUMMY_INPUTS,
     ModelOutput,
     add_code_sample_docstrings,
@@ -14,7 +13,7 @@ from .file_utils import (
     add_start_docstrings_to_model_forward,
     replace_return_docstrings,
 )
-from .modeling_tf_outputs import (
+from ...modeling_tf_outputs import (
     TFBaseModelOutput,
     TFMaskedLMOutput,
     TFMultipleChoiceModelOutput,
@@ -22,7 +21,7 @@ from .modeling_tf_outputs import (
     TFSequenceClassifierOutput,
     TFTokenClassifierOutput,
 )
-from .modeling_tf_utils import (
+from ...modeling_tf_utils import (
     TFMaskedLanguageModelingLoss,
     TFMultipleChoiceLoss,
     TFPreTrainedModel,
@@ -34,8 +33,9 @@ from .modeling_tf_utils import (
     keras_serializable,
     shape_list,
 )
-from .tokenization_utils import BatchEncoding
-from .utils import logging
+from ...tokenization_utils import BatchEncoding
+from ...utils import logging
+from .configuration_electra import ElectraConfig
 
 
 logger = logging.get_logger(__name__)
@@ -54,7 +54,7 @@ TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST = [
 ]
 
 
-# Copied from transformers.modeling_tf_bert.TFBertSelfAttention
+# Copied from transformers.models.bert.modeling_tf_bert.TFBertSelfAttention
 class TFElectraSelfAttention(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
         super().__init__(**kwargs)
@@ -126,7 +126,7 @@ class TFElectraSelfAttention(tf.keras.layers.Layer):
         return outputs
 
 
-# Copied from transformers.modeling_tf_bert.TFBertSelfOutput
+# Copied from transformers.models.bert.modeling_tf_bert.TFBertSelfOutput
 class TFElectraSelfOutput(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
         super().__init__(**kwargs)
@@ -145,7 +145,7 @@ class TFElectraSelfOutput(tf.keras.layers.Layer):
         return hidden_states
 
 
-# Copied from from transformers.modeling_tf_bert.TFBertAttention with Bert->Electra
+# Copied from from transformers.models.bert.modeling_tf_bert.TFBertAttention with Bert->Electra
 class TFElectraAttention(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
         super().__init__(**kwargs)
@@ -166,7 +166,7 @@ class TFElectraAttention(tf.keras.layers.Layer):
         return outputs
 
 
-# Copied from transformers.modeling_tf_bert.TFBertIntermediate
+# Copied from transformers.models.bert.modeling_tf_bert.TFBertIntermediate
 class TFElectraIntermediate(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
         super().__init__(**kwargs)
@@ -187,7 +187,7 @@ class TFElectraIntermediate(tf.keras.layers.Layer):
         return hidden_states
 
 
-# Copied from transformers.modeling_tf_bert.TFBertOutput
+# Copied from transformers.models.bert.modeling_tf_bert.TFBertOutput
 class TFElectraOutput(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
         super().__init__(**kwargs)
@@ -206,7 +206,7 @@ class TFElectraOutput(tf.keras.layers.Layer):
         return hidden_states
 
 
-# Copied from transformers.modeling_tf_bert.TFBertLayer with Bert->Electra
+# Copied from transformers.models.bert.modeling_tf_bert.TFBertLayer with Bert->Electra
 class TFElectraLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
         super().__init__(**kwargs)
@@ -227,7 +227,7 @@ class TFElectraLayer(tf.keras.layers.Layer):
         return outputs
 
 
-# Copied from transformers.modeling_tf_bert.TFBertEncoder with Bert->Electra
+# Copied from transformers.models.bert.modeling_tf_bert.TFBertEncoder with Bert->Electra
 class TFElectraEncoder(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
         super().__init__(**kwargs)
@@ -271,7 +271,7 @@ class TFElectraEncoder(tf.keras.layers.Layer):
         )
 
 
-# Copied from transformers.modeling_tf_bert.TFBertPooler
+# Copied from transformers.models.bert.modeling_tf_bert.TFBertPooler
 class TFElectraPooler(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
         super().__init__(**kwargs)
@@ -332,7 +332,7 @@ class TFElectraEmbeddings(tf.keras.layers.Layer):
 
         super().build(input_shape)
 
-    # Copied from transformers.modeling_tf_bert.TFBertEmbeddings.call
+    # Copied from transformers.models.bert.modeling_tf_bert.TFBertEmbeddings.call
     def call(
         self,
         input_ids=None,
@@ -367,7 +367,7 @@ class TFElectraEmbeddings(tf.keras.layers.Layer):
         else:
             raise ValueError("mode {} is not valid.".format(mode))
 
-    # Copied from transformers.modeling_tf_bert.TFBertEmbeddings._embedding
+    # Copied from transformers.models.bert.modeling_tf_bert.TFBertEmbeddings._embedding
     def _embedding(self, input_ids, position_ids, token_type_ids, inputs_embeds, training=False):
         """Applies embedding based on inputs tensor."""
         assert not (input_ids is None and inputs_embeds is None)

src/transformers/tokenization_electra.py → src/transformers/models/electra/tokenization_electra.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .tokenization_bert import BertTokenizer
+from ..bert.tokenization_bert import BertTokenizer
 
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}

src/transformers/tokenization_electra_fast.py → src/transformers/models/electra/tokenization_electra_fast.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .tokenization_bert_fast import BertTokenizerFast
+from ..bert.tokenization_bert_fast import BertTokenizerFast
 from .tokenization_electra import ElectraTokenizer
 
 

src/transformers/models/encoder_decoder/__init__.py

+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all.
+
+from ...file_utils import is_torch_available
+from .configuration_encoder_decoder import EncoderDecoderConfig
+
+
+if is_torch_available():
+    from .modeling_encoder_decoder import EncoderDecoderModel

src/transformers/configuration_encoder_decoder.py → src/transformers/models/encoder_decoder/configuration_encoder_decoder.py

@@ -16,8 +16,8 @@
 
 import copy
 
-from .configuration_utils import PretrainedConfig
-from .utils import logging
+from ...configuration_utils import PretrainedConfig
+from ...utils import logging
 
 
 logger = logging.get_logger(__name__)
@@ -81,7 +81,7 @@ class EncoderDecoderConfig(PretrainedConfig):
         decoder_config = kwargs.pop("decoder")
         decoder_model_type = decoder_config.pop("model_type")
 
-        from .configuration_auto import AutoConfig
+        from ..auto.configuration_auto import AutoConfig
 
         self.encoder = AutoConfig.for_model(encoder_model_type, **encoder_config)
         self.decoder = AutoConfig.for_model(decoder_model_type, **decoder_config)

src/transformers/modeling_encoder_decoder.py → src/transformers/models/encoder_decoder/modeling_encoder_decoder.py

@@ -17,12 +17,12 @@
 
 from typing import Optional
 
+from ...configuration_utils import PretrainedConfig
+from ...file_utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings
+from ...modeling_outputs import Seq2SeqLMOutput
+from ...modeling_utils import PreTrainedModel
+from ...utils import logging
 from .configuration_encoder_decoder import EncoderDecoderConfig
-from .configuration_utils import PretrainedConfig
-from .file_utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings
-from .modeling_outputs import Seq2SeqLMOutput
-from .modeling_utils import PreTrainedModel
-from .utils import logging
 
 
 logger = logging.get_logger(__name__)
@@ -155,12 +155,12 @@ class EncoderDecoderModel(PreTrainedModel):
         super().__init__(config)
 
         if encoder is None:
-            from .modeling_auto import AutoModel
+            from ..auto.modeling_auto import AutoModel
 
             encoder = AutoModel.from_config(config.encoder)
 
         if decoder is None:
-            from .modeling_auto import AutoModelForCausalLM
+            from ..auto.modeling_auto import AutoModelForCausalLM
 
             decoder = AutoModelForCausalLM.from_config(config.decoder)
 
@@ -286,10 +286,10 @@ class EncoderDecoderModel(PreTrainedModel):
             assert (
                 encoder_pretrained_model_name_or_path is not None
             ), "If `model` is not defined as an argument, a `encoder_pretrained_model_name_or_path` has to be defined"
-            from .modeling_auto import AutoModel
+            from ..auto.modeling_auto import AutoModel
 
             if "config" not in kwargs_encoder:
-                from .configuration_auto import AutoConfig
+                from ..auto.configuration_auto import AutoConfig
 
                 encoder_config = AutoConfig.from_pretrained(encoder_pretrained_model_name_or_path)
                 if encoder_config.is_decoder is True or encoder_config.add_cross_attention is True:
@@ -309,10 +309,10 @@ class EncoderDecoderModel(PreTrainedModel):
             assert (
                 decoder_pretrained_model_name_or_path is not None
             ), "If `decoder_model` is not defined as an argument, a `decoder_pretrained_model_name_or_path` has to be defined"
-            from .modeling_auto import AutoModelForCausalLM
+            from ..auto.modeling_auto import AutoModelForCausalLM
 
             if "config" not in kwargs_decoder:
-                from .configuration_auto import AutoConfig
+                from ..auto.configuration_auto import AutoConfig
 
                 decoder_config = AutoConfig.from_pretrained(decoder_pretrained_model_name_or_path)
                 if decoder_config.is_decoder is False or decoder_config.add_cross_attention is False:

src/transformers/models/flaubert/__init__.py

+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all.
+
+from ...file_utils import is_tf_available, is_torch_available
+from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig
+from .tokenization_flaubert import FlaubertTokenizer
+
+
+if is_torch_available():
+    from .modeling_flaubert import (
+        FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+        FlaubertForMultipleChoice,
+        FlaubertForQuestionAnswering,
+        FlaubertForQuestionAnsweringSimple,
+        FlaubertForSequenceClassification,
+        FlaubertForTokenClassification,
+        FlaubertModel,
+        FlaubertWithLMHeadModel,
+    )
+
+if is_tf_available():
+    from .modeling_tf_flaubert import (
+        TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+        TFFlaubertForMultipleChoice,
+        TFFlaubertForQuestionAnsweringSimple,
+        TFFlaubertForSequenceClassification,
+        TFFlaubertForTokenClassification,
+        TFFlaubertModel,
+        TFFlaubertWithLMHeadModel,
+    )

src/transformers/configuration_flaubert.py → src/transformers/models/flaubert/configuration_flaubert.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 """ Flaubert configuration, based on XLM. """
 
-from .configuration_xlm import XLMConfig
-from .utils import logging
+from ...utils import logging
+from ..xlm.configuration_xlm import XLMConfig
 
 
 logger = logging.get_logger(__name__)

src/transformers/modeling_flaubert.py → src/transformers/models/flaubert/modeling_flaubert.py

@@ -20,10 +20,10 @@ import random
 import torch
 from torch.nn import functional as F
 
-from .configuration_flaubert import FlaubertConfig
-from .file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
-from .modeling_outputs import BaseModelOutput
-from .modeling_xlm import (
+from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
+from ...modeling_outputs import BaseModelOutput
+from ...utils import logging
+from ..xlm.modeling_xlm import (
     XLMForMultipleChoice,
     XLMForQuestionAnswering,
     XLMForQuestionAnsweringSimple,
@@ -33,7 +33,7 @@ from .modeling_xlm import (
     XLMWithLMHeadModel,
     get_masks,
 )
-from .utils import logging
+from .configuration_flaubert import FlaubertConfig
 
 
 logger = logging.get_logger(__name__)

src/transformers/modeling_tf_flaubert.py → src/transformers/models/flaubert/modeling_tf_flaubert.py

@@ -24,23 +24,23 @@ import tensorflow as tf
 
 from transformers.activations_tf import get_tf_activation
 
-from .configuration_flaubert import FlaubertConfig
-from .file_utils import (
+from ...file_utils import (
     ModelOutput,
     add_code_sample_docstrings,
     add_start_docstrings,
     add_start_docstrings_to_model_forward,
 )
-from .modeling_tf_outputs import TFBaseModelOutput
-from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, get_initializer, keras_serializable, shape_list
-from .modeling_tf_xlm import (
+from ...modeling_tf_outputs import TFBaseModelOutput
+from ...modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, get_initializer, keras_serializable, shape_list
+from ...tokenization_utils import BatchEncoding
+from ...utils import logging
+from ..xlm.modeling_tf_xlm import (
     TFXLMForMultipleChoice,
     TFXLMForQuestionAnsweringSimple,
     TFXLMForSequenceClassification,
     TFXLMForTokenClassification,
 )
-from .tokenization_utils import BatchEncoding
-from .utils import logging
+from .configuration_flaubert import FlaubertConfig
 
 
 logger = logging.get_logger(__name__)
@@ -234,7 +234,7 @@ class TFFlaubertModel(TFFlaubertPreTrainedModel):
         return outputs
 
 
-# Copied from transformers.modeling_tf_xlm.TFXLMMultiHeadAttention with XLM->Flaubert
+# Copied from transformers.models.xlm.modeling_tf_xlm.TFXLMMultiHeadAttention with XLM->Flaubert
 class TFFlaubertMultiHeadAttention(tf.keras.layers.Layer):
     NEW_ID = itertools.count()
 
@@ -328,7 +328,7 @@ class TFFlaubertMultiHeadAttention(tf.keras.layers.Layer):
         return outputs
 
 
-# Copied from transformers.modeling_tf_xlm.TFXLMTransformerFFN
+# Copied from transformers.models.xlm.modeling_tf_xlm.TFXLMTransformerFFN
 class TFFlaubertTransformerFFN(tf.keras.layers.Layer):
     def __init__(self, in_dim, dim_hidden, out_dim, config, **kwargs):
         super().__init__(**kwargs)
@@ -632,7 +632,7 @@ class TFFlaubertMainLayer(tf.keras.layers.Layer):
         return TFBaseModelOutput(last_hidden_state=tensor, hidden_states=hidden_states, attentions=attentions)
 
 
-# Copied from transformers.modeling_tf_xlm.TFXLMPredLayer
+# Copied from transformers.models.xlm.modeling_tf_xlm.TFXLMPredLayer
 class TFFlaubertPredLayer(tf.keras.layers.Layer):
     """
     Prediction layer (cross_entropy or adaptive_softmax).

src/transformers/tokenization_flaubert.py → src/transformers/models/flaubert/tokenization_flaubert.py

@@ -19,8 +19,8 @@ import unicodedata
 
 import six
 
-from .tokenization_xlm import XLMTokenizer
-from .utils import logging
+from ...utils import logging
+from ..xlm.tokenization_xlm import XLMTokenizer
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/fsmt/__init__.py

+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all.
+
+from ...file_utils import is_torch_available
+from .configuration_fsmt import FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP, FSMTConfig
+from .tokenization_fsmt import FSMTTokenizer
+
+
+if is_torch_available():
+    from .modeling_fsmt import FSMTForConditionalGeneration, FSMTModel, PretrainedFSMTModel

src/transformers/configuration_fsmt.py → src/transformers/models/fsmt/configuration_fsmt.py

@@ -17,8 +17,8 @@
 
 import copy
 
-from .configuration_utils import PretrainedConfig
-from .utils import logging
+from ...configuration_utils import PretrainedConfig
+from ...utils import logging
 
 
 logger = logging.get_logger(__name__)

src/transformers/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py → src/transformers/models/fsmt/convert_fsmt_original_pytorch_checkpoint_to_pytorch.py

@@ -32,9 +32,7 @@ from fairseq import hub_utils
 from fairseq.data.dictionary import Dictionary
 
 from transformers import WEIGHTS_NAME, logging
-from transformers.configuration_fsmt import FSMTConfig
-from transformers.modeling_fsmt import FSMTForConditionalGeneration
-from transformers.tokenization_fsmt import VOCAB_FILES_NAMES
+from transformers.models.fsmt import VOCAB_FILES_NAMES, FSMTConfig, FSMTForConditionalGeneration
 from transformers.tokenization_utils_base import TOKENIZER_CONFIG_FILE
 
 

src/transformers/modeling_fsmt.py → src/transformers/models/fsmt/modeling_fsmt.py

@@ -37,23 +37,23 @@ import torch.nn.functional as F
 from torch import Tensor, nn
 from torch.nn import CrossEntropyLoss
 
-from .activations import ACT2FN
-from .configuration_fsmt import FSMTConfig
-from .file_utils import (
+from ...activations import ACT2FN
+from ...file_utils import (
     add_code_sample_docstrings,
     add_end_docstrings,
     add_start_docstrings,
     add_start_docstrings_to_model_forward,
     replace_return_docstrings,
 )
-from .modeling_outputs import (
+from ...modeling_outputs import (
     BaseModelOutput,
     BaseModelOutputWithPastAndCrossAttentions,
     Seq2SeqLMOutput,
     Seq2SeqModelOutput,
 )
-from .modeling_utils import PreTrainedModel
-from .utils import logging
+from ...modeling_utils import PreTrainedModel
+from ...utils import logging
+from .configuration_fsmt import FSMTConfig
 
 
 logger = logging.get_logger(__name__)

src/transformers/tokenization_fsmt.py → src/transformers/models/fsmt/tokenization_fsmt.py

@@ -23,10 +23,10 @@ from typing import Dict, List, Optional, Tuple
 
 import sacremoses as sm
 
-from .file_utils import add_start_docstrings
-from .tokenization_utils import BatchEncoding, PreTrainedTokenizer
-from .tokenization_utils_base import PREPARE_SEQ2SEQ_BATCH_DOCSTRING
-from .utils import logging
+from ...file_utils import add_start_docstrings
+from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
+from ...tokenization_utils_base import PREPARE_SEQ2SEQ_BATCH_DOCSTRING
+from ...utils import logging
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/funnel/__init__.py

+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all.
+
+from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
+from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
+from .tokenization_funnel import FunnelTokenizer
+
+
+if is_tokenizers_available():
+    from .tokenization_funnel_fast import FunnelTokenizerFast
+
+if is_torch_available():
+    from .modeling_funnel import (
+        FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
+        FunnelBaseModel,
+        FunnelForMaskedLM,
+        FunnelForMultipleChoice,
+        FunnelForPreTraining,
+        FunnelForQuestionAnswering,
+        FunnelForSequenceClassification,
+        FunnelForTokenClassification,
+        FunnelModel,
+        load_tf_weights_in_funnel,
+    )
+
+if is_tf_available():
+    from .modeling_tf_funnel import (
+        TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
+        TFFunnelBaseModel,
+        TFFunnelForMaskedLM,
+        TFFunnelForMultipleChoice,
+        TFFunnelForPreTraining,
+        TFFunnelForQuestionAnswering,
+        TFFunnelForSequenceClassification,
+        TFFunnelForTokenClassification,
+        TFFunnelModel,
+    )

src/transformers/configuration_funnel.py → src/transformers/models/funnel/configuration_funnel.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 """ Funnel Transformer model configuration """
 
-from .configuration_utils import PretrainedConfig
-from .utils import logging
+from ...configuration_utils import PretrainedConfig
+from ...utils import logging
 
 
 logger = logging.get_logger(__name__)

src/transformers/modeling_funnel.py → src/transformers/models/funnel/modeling_funnel.py

@@ -24,16 +24,15 @@ from torch import nn
 from torch.nn import CrossEntropyLoss, MSELoss
 from torch.nn import functional as F
 
-from .activations import ACT2FN
-from .configuration_funnel import FunnelConfig
-from .file_utils import (
+from ...activations import ACT2FN
+from ...file_utils import (
     ModelOutput,
     add_code_sample_docstrings,
     add_start_docstrings,
     add_start_docstrings_to_model_forward,
     replace_return_docstrings,
 )
-from .modeling_outputs import (
+from ...modeling_outputs import (
     BaseModelOutput,
     MaskedLMOutput,
     MultipleChoiceModelOutput,
@@ -41,8 +40,9 @@ from .modeling_outputs import (
     SequenceClassifierOutput,
     TokenClassifierOutput,
 )
-from .modeling_utils import PreTrainedModel
-from .utils import logging
+from ...modeling_utils import PreTrainedModel
+from ...utils import logging
+from .configuration_funnel import FunnelConfig
 
 
 logger = logging.get_logger(__name__)