Reorganize repo (#8580)

* Put models in subfolders

* Styling

* Fix imports in tests

* More fixes in test imports

* Sneaky hidden imports

* Fix imports in doc files

* More sneaky imports

* Finish fixing tests

* Fix examples

* Fix path for copies

* More fixes for examples

* Fix dummy files

* More fixes for example

* More model import fixes

* Is this why you're unhappy GitHub?

* Fix imports in conver command

src/transformers/tokenization_deberta.py → src/transformers/models/deberta/tokenization_deberta.py

@@ -26,8 +26,8 @@ import tqdm
 
 import requests
 
-from .tokenization_utils import PreTrainedTokenizer
-from .utils import logging
+from ...tokenization_utils import PreTrainedTokenizer
+from ...utils import logging
 
 
 try:

src/transformers/models/distilbert/__init__.py

+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all.
+
+from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
+from .configuration_distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig
+from .tokenization_distilbert import DistilBertTokenizer
+
+
+if is_tokenizers_available():
+    from .tokenization_distilbert_fast import DistilBertTokenizerFast
+
+if is_torch_available():
+    from .modeling_distilbert import (
+        DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+        DistilBertForMaskedLM,
+        DistilBertForMultipleChoice,
+        DistilBertForQuestionAnswering,
+        DistilBertForSequenceClassification,
+        DistilBertForTokenClassification,
+        DistilBertModel,
+        DistilBertPreTrainedModel,
+    )
+
+if is_tf_available():
+    from .modeling_tf_distilbert import (
+        TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+        TFDistilBertForMaskedLM,
+        TFDistilBertForMultipleChoice,
+        TFDistilBertForQuestionAnswering,
+        TFDistilBertForSequenceClassification,
+        TFDistilBertForTokenClassification,
+        TFDistilBertMainLayer,
+        TFDistilBertModel,
+        TFDistilBertPreTrainedModel,
+    )

src/transformers/configuration_distilbert.py → src/transformers/models/distilbert/configuration_distilbert.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 """ DistilBERT model configuration """
 
-from .configuration_utils import PretrainedConfig
-from .utils import logging
+from ...configuration_utils import PretrainedConfig
+from ...utils import logging
 
 
 logger = logging.get_logger(__name__)

src/transformers/modeling_distilbert.py → src/transformers/models/distilbert/modeling_distilbert.py

@@ -27,15 +27,14 @@ import torch
 import torch.nn as nn
 from torch.nn import CrossEntropyLoss
 
-from .activations import gelu
-from .configuration_distilbert import DistilBertConfig
-from .file_utils import (
+from ...activations import gelu
+from ...file_utils import (
     add_code_sample_docstrings,
     add_start_docstrings,
     add_start_docstrings_to_model_forward,
     replace_return_docstrings,
 )
-from .modeling_outputs import (
+from ...modeling_outputs import (
     BaseModelOutput,
     MaskedLMOutput,
     MultipleChoiceModelOutput,
@@ -43,13 +42,14 @@ from .modeling_outputs import (
     SequenceClassifierOutput,
     TokenClassifierOutput,
 )
-from .modeling_utils import (
+from ...modeling_utils import (
     PreTrainedModel,
     apply_chunking_to_forward,
     find_pruneable_heads_and_indices,
     prune_linear_layer,
 )
-from .utils import logging
+from ...utils import logging
+from .configuration_distilbert import DistilBertConfig
 
 
 logger = logging.get_logger(__name__)

src/transformers/modeling_tf_distilbert.py → src/transformers/models/distilbert/modeling_tf_distilbert.py

@@ -19,15 +19,14 @@
 
 import tensorflow as tf
 
-from .activations_tf import get_tf_activation
-from .configuration_distilbert import DistilBertConfig
-from .file_utils import (
+from ...activations_tf import get_tf_activation
+from ...file_utils import (
     MULTIPLE_CHOICE_DUMMY_INPUTS,
     add_code_sample_docstrings,
     add_start_docstrings,
     add_start_docstrings_to_model_forward,
 )
-from .modeling_tf_outputs import (
+from ...modeling_tf_outputs import (
     TFBaseModelOutput,
     TFMaskedLMOutput,
     TFMultipleChoiceModelOutput,
@@ -35,7 +34,7 @@ from .modeling_tf_outputs import (
     TFSequenceClassifierOutput,
     TFTokenClassifierOutput,
 )
-from .modeling_tf_utils import (
+from ...modeling_tf_utils import (
     TFMaskedLanguageModelingLoss,
     TFMultipleChoiceLoss,
     TFPreTrainedModel,
@@ -47,8 +46,9 @@ from .modeling_tf_utils import (
     keras_serializable,
     shape_list,
 )
-from .tokenization_utils import BatchEncoding
-from .utils import logging
+from ...tokenization_utils import BatchEncoding
+from ...utils import logging
+from .configuration_distilbert import DistilBertConfig
 
 
 logger = logging.get_logger(__name__)

src/transformers/tokenization_distilbert.py → src/transformers/models/distilbert/tokenization_distilbert.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 """Tokenization classes for DistilBERT."""
 
-from .tokenization_bert import BertTokenizer
-from .utils import logging
+from ...utils import logging
+from ..bert.tokenization_bert import BertTokenizer
 
 
 logger = logging.get_logger(__name__)

src/transformers/tokenization_distilbert_fast.py → src/transformers/models/distilbert/tokenization_distilbert_fast.py

@@ -14,9 +14,9 @@
 # limitations under the License.
 """Tokenization classes for DistilBERT."""
 
-from .tokenization_bert_fast import BertTokenizerFast
+from ...utils import logging
+from ..bert.tokenization_bert_fast import BertTokenizerFast
 from .tokenization_distilbert import DistilBertTokenizer
-from .utils import logging
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/dpr/__init__.py

+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all.
+
+from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
+from .configuration_dpr import DPR_PRETRAINED_CONFIG_ARCHIVE_MAP, DPRConfig
+from .tokenization_dpr import (
+    DPRContextEncoderTokenizer,
+    DPRQuestionEncoderTokenizer,
+    DPRReaderOutput,
+    DPRReaderTokenizer,
+)
+
+
+if is_tokenizers_available():
+    from .tokenization_dpr_fast import (
+        DPRContextEncoderTokenizerFast,
+        DPRQuestionEncoderTokenizerFast,
+        DPRReaderTokenizerFast,
+    )
+
+if is_torch_available():
+    from .modeling_dpr import (
+        DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
+        DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
+        DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
+        DPRContextEncoder,
+        DPRPretrainedContextEncoder,
+        DPRPretrainedQuestionEncoder,
+        DPRPretrainedReader,
+        DPRQuestionEncoder,
+        DPRReader,
+    )
+
+if is_tf_available():
+    from .modeling_tf_dpr import (
+        TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
+        TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
+        TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
+        TFDPRContextEncoder,
+        TFDPRPretrainedContextEncoder,
+        TFDPRPretrainedQuestionEncoder,
+        TFDPRPretrainedReader,
+        TFDPRQuestionEncoder,
+        TFDPRReader,
+    )

src/transformers/configuration_dpr.py → src/transformers/models/dpr/configuration_dpr.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 """ DPR model configuration """
 
-from .configuration_utils import PretrainedConfig
-from .utils import logging
+from ...configuration_utils import PretrainedConfig
+from ...utils import logging
 
 
 logger = logging.get_logger(__name__)

src/transformers/modeling_dpr.py → src/transformers/models/dpr/modeling_dpr.py

@@ -21,17 +21,17 @@ from typing import Optional, Tuple, Union
 import torch
 from torch import Tensor, nn
 
-from .configuration_dpr import DPRConfig
-from .file_utils import (
+from ...file_utils import (
     ModelOutput,
     add_start_docstrings,
     add_start_docstrings_to_model_forward,
     replace_return_docstrings,
 )
-from .modeling_bert import BertModel
-from .modeling_outputs import BaseModelOutputWithPooling
-from .modeling_utils import PreTrainedModel
-from .utils import logging
+from ...modeling_outputs import BaseModelOutputWithPooling
+from ...modeling_utils import PreTrainedModel
+from ...utils import logging
+from ..bert.modeling_bert import BertModel
+from .configuration_dpr import DPRConfig
 
 
 logger = logging.get_logger(__name__)

src/transformers/modeling_tf_dpr.py → src/transformers/models/dpr/modeling_tf_dpr.py

@@ -22,18 +22,18 @@ import tensorflow as tf
 from tensorflow import Tensor
 from tensorflow.keras.layers import Dense
 
-from .configuration_dpr import DPRConfig
-from .file_utils import (
+from ...file_utils import (
     ModelOutput,
     add_start_docstrings,
     add_start_docstrings_to_model_forward,
     replace_return_docstrings,
 )
-from .modeling_tf_bert import TFBertMainLayer
-from .modeling_tf_outputs import TFBaseModelOutputWithPooling
-from .modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list
-from .tokenization_utils import BatchEncoding
-from .utils import logging
+from ...modeling_tf_outputs import TFBaseModelOutputWithPooling
+from ...modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list
+from ...tokenization_utils import BatchEncoding
+from ...utils import logging
+from ..bert.modeling_tf_bert import TFBertMainLayer
+from .configuration_dpr import DPRConfig
 
 
 logger = logging.get_logger(__name__)

src/transformers/tokenization_dpr.py → src/transformers/models/dpr/tokenization_dpr.py

@@ -18,10 +18,10 @@
 import collections
 from typing import List, Optional, Union
 
-from .file_utils import add_end_docstrings, add_start_docstrings
-from .tokenization_bert import BertTokenizer
-from .tokenization_utils_base import BatchEncoding, TensorType
-from .utils import logging
+from ...file_utils import add_end_docstrings, add_start_docstrings
+from ...tokenization_utils_base import BatchEncoding, TensorType
+from ...utils import logging
+from ..bert.tokenization_bert import BertTokenizer
 
 
 logger = logging.get_logger(__name__)

src/transformers/tokenization_dpr_fast.py → src/transformers/models/dpr/tokenization_dpr_fast.py

@@ -18,11 +18,11 @@
 import collections
 from typing import List, Optional, Union
 
-from .file_utils import add_end_docstrings, add_start_docstrings
-from .tokenization_bert_fast import BertTokenizerFast
+from ...file_utils import add_end_docstrings, add_start_docstrings
+from ...tokenization_utils_base import BatchEncoding, TensorType
+from ...utils import logging
+from ..bert.tokenization_bert_fast import BertTokenizerFast
 from .tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer, DPRReaderTokenizer
-from .tokenization_utils_base import BatchEncoding, TensorType
-from .utils import logging
 
 
 logger = logging.get_logger(__name__)

src/transformers/models/electra/__init__.py

+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all.
+
+from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
+from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig
+from .tokenization_electra import ElectraTokenizer
+
+
+if is_tokenizers_available():
+    from .tokenization_electra_fast import ElectraTokenizerFast
+
+if is_torch_available():
+    from .modeling_electra import (
+        ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
+        ElectraForMaskedLM,
+        ElectraForMultipleChoice,
+        ElectraForPreTraining,
+        ElectraForQuestionAnswering,
+        ElectraForSequenceClassification,
+        ElectraForTokenClassification,
+        ElectraModel,
+        ElectraPreTrainedModel,
+        load_tf_weights_in_electra,
+    )
+
+if is_tf_available():
+    from .modeling_tf_electra import (
+        TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
+        TFElectraForMaskedLM,
+        TFElectraForMultipleChoice,
+        TFElectraForPreTraining,
+        TFElectraForQuestionAnswering,
+        TFElectraForSequenceClassification,
+        TFElectraForTokenClassification,
+        TFElectraModel,
+        TFElectraPreTrainedModel,
+    )

src/transformers/configuration_electra.py → src/transformers/models/electra/configuration_electra.py

@@ -15,8 +15,8 @@
 # limitations under the License.
 """ ELECTRA model configuration """
 
-from .configuration_utils import PretrainedConfig
-from .utils import logging
+from ...configuration_utils import PretrainedConfig
+from ...utils import logging
 
 
 logger = logging.get_logger(__name__)

src/transformers/modeling_electra.py → src/transformers/models/electra/modeling_electra.py

@@ -24,16 +24,15 @@ import torch
 import torch.nn as nn
 from torch.nn import CrossEntropyLoss, MSELoss
 
-from .activations import ACT2FN, get_activation
-from .configuration_electra import ElectraConfig
-from .file_utils import (
+from ...activations import ACT2FN, get_activation
+from ...file_utils import (
     ModelOutput,
     add_code_sample_docstrings,
     add_start_docstrings,
     add_start_docstrings_to_model_forward,
     replace_return_docstrings,
 )
-from .modeling_outputs import (
+from ...modeling_outputs import (
     BaseModelOutputWithCrossAttentions,
     MaskedLMOutput,
     MultipleChoiceModelOutput,
@@ -41,14 +40,15 @@ from .modeling_outputs import (
     SequenceClassifierOutput,
     TokenClassifierOutput,
 )
-from .modeling_utils import (
+from ...modeling_utils import (
     PreTrainedModel,
     SequenceSummary,
     apply_chunking_to_forward,
     find_pruneable_heads_and_indices,
     prune_linear_layer,
 )
-from .utils import logging
+from ...utils import logging
+from .configuration_electra import ElectraConfig
 
 
 logger = logging.get_logger(__name__)
@@ -167,7 +167,7 @@ class ElectraEmbeddings(nn.Module):
         # position_ids (1, len position emb) is contiguous in memory and exported when serialized
         self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)))
 
-    # Copied from transformers.modeling_bert.BertEmbeddings.forward
+    # Copied from transformers.models.bert.modeling_bert.BertEmbeddings.forward
     def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None):
         if input_ids is not None:
             input_shape = input_ids.size()
@@ -193,7 +193,7 @@ class ElectraEmbeddings(nn.Module):
         return embeddings
 
 
-# Copied from transformers.modeling_bert.BertSelfAttention with Bert->Electra
+# Copied from transformers.models.bert.modeling_bert.BertSelfAttention with Bert->Electra
 class ElectraSelfAttention(nn.Module):
     def __init__(self, config):
         super().__init__()
@@ -272,7 +272,7 @@ class ElectraSelfAttention(nn.Module):
         return outputs
 
 
-# Copied from transformers.modeling_bert.BertSelfOutput
+# Copied from transformers.models.bert.modeling_bert.BertSelfOutput
 class ElectraSelfOutput(nn.Module):
     def __init__(self, config):
         super().__init__()
@@ -287,7 +287,7 @@ class ElectraSelfOutput(nn.Module):
         return hidden_states
 
 
-# Copied from transformers.modeling_bert.BertAttention with Bert->Electra
+# Copied from transformers.models.bert.modeling_bert.BertAttention with Bert->Electra
 class ElectraAttention(nn.Module):
     def __init__(self, config):
         super().__init__()
@@ -335,7 +335,7 @@ class ElectraAttention(nn.Module):
         return outputs
 
 
-# Copied from transformers.modeling_bert.BertIntermediate
+# Copied from transformers.models.bert.modeling_bert.BertIntermediate
 class ElectraIntermediate(nn.Module):
     def __init__(self, config):
         super().__init__()
@@ -351,7 +351,7 @@ class ElectraIntermediate(nn.Module):
         return hidden_states
 
 
-# Copied from transformers.modeling_bert.BertOutput
+# Copied from transformers.models.bert.modeling_bert.BertOutput
 class ElectraOutput(nn.Module):
     def __init__(self, config):
         super().__init__()
@@ -366,7 +366,7 @@ class ElectraOutput(nn.Module):
         return hidden_states
 
 
-# Copied from transformers.modeling_bert.BertLayer with Bert->Electra
+# Copied from transformers.models.bert.modeling_bert.BertLayer with Bert->Electra
 class ElectraLayer(nn.Module):
     def __init__(self, config):
         super().__init__()
@@ -426,7 +426,7 @@ class ElectraLayer(nn.Module):
         return layer_output
 
 
-# Copied from transformers.modeling_bert.BertEncoder with Bert->Electra
+# Copied from transformers.models.bert.modeling_bert.BertEncoder with Bert->Electra
 class ElectraEncoder(nn.Module):
     def __init__(self, config):
         super().__init__()
@@ -548,7 +548,7 @@ class ElectraPreTrainedModel(PreTrainedModel):
     authorized_missing_keys = [r"position_ids"]
     authorized_unexpected_keys = [r"electra\.embeddings_project\.weight", r"electra\.embeddings_project\.bias"]
 
-    # Copied from transformers.modeling_bert.BertPreTrainedModel._init_weights
+    # Copied from transformers.models.bert.modeling_bert.BertPreTrainedModel._init_weights
     def _init_weights(self, module):
         """ Initialize the weights """
         if isinstance(module, (nn.Linear, nn.Embedding)):