Transformers fast import part 2 (#9446)

* Main init work

* Add version

* Change from absolute to relative imports

* Fix imports

* One more typo

* More typos

* Styling

* Make quality script pass

* Add necessary replace in template

* Fix typos

* Spaces are ignored in replace for some reason

* Forgot one models.

* Fixes for import
Co-authored-by: LysandreJik <lysandre.debut@reseau.eseo.fr>

* Add documentation

* Styling
Co-authored-by: LysandreJik <lysandre.debut@reseau.eseo.fr>

src/transformers/models/mobilebert/convert_mobilebert_original_tf_checkpoint_to_pytorch.py

@@ -16,8 +16,8 @@ import argparse
 
 import torch
 
-from transformers import MobileBertConfig, MobileBertForPreTraining, load_tf_weights_in_mobilebert
-from transformers.utils import logging
+from ...utils import logging
+from . import MobileBertConfig, MobileBertForPreTraining, load_tf_weights_in_mobilebert
 
 
 logging.set_verbosity_info()

src/transformers/models/openai/convert_openai_original_tf_checkpoint_to_pytorch.py

@@ -19,8 +19,9 @@ import argparse
 
 import torch
 
-from transformers import CONFIG_NAME, WEIGHTS_NAME, OpenAIGPTConfig, OpenAIGPTModel, load_tf_weights_in_openai_gpt
-from transformers.utils import logging
+from ...file_utils import CONFIG_NAME, WEIGHTS_NAME
+from ...utils import logging
+from . import OpenAIGPTConfig, OpenAIGPTModel, load_tf_weights_in_openai_gpt
 
 
 logging.set_verbosity_info()

src/transformers/models/pegasus/convert_pegasus_tf_to_pytorch.py

@@ -22,8 +22,8 @@ import tensorflow as tf
 import torch
 from tqdm import tqdm
 
-from transformers import PegasusConfig, PegasusForConditionalGeneration, PegasusTokenizer
-from transformers.models.pegasus.configuration_pegasus import DEFAULTS, task_specific_params
+from . import PegasusConfig, PegasusForConditionalGeneration, PegasusTokenizer
+from .configuration_pegasus import DEFAULTS, task_specific_params
 
 
 PATTERNS = [

src/transformers/models/prophetnet/convert_prophetnet_original_pytorch_checkpoint_to_pytorch.py

@@ -19,8 +19,6 @@ import argparse
 
 import torch
 
-from transformers import ProphetNetForConditionalGeneration, XLMProphetNetForConditionalGeneration, logging
-
 # transformers_old should correspond to branch `save_old_prophetnet_model_structure` here
 # original prophetnet_checkpoints are saved under `patrickvonplaten/..._old` respectively
 from transformers_old.modeling_prophetnet import (
@@ -30,6 +28,8 @@ from transformers_old.modeling_xlm_prophetnet import (
     XLMProphetNetForConditionalGeneration as XLMProphetNetForConditionalGenerationOld,
 )
 
+from . import ProphetNetForConditionalGeneration, XLMProphetNetForConditionalGeneration, logging
+
 
 logger = logging.get_logger(__name__)
 logging.set_verbosity_info()

src/transformers/models/reformer/convert_reformer_trax_checkpoint_to_pytorch.py

@@ -21,8 +21,8 @@ import pickle
 import numpy as np
 import torch
 
-from transformers import ReformerConfig, ReformerModelWithLMHead
-from transformers.utils import logging
+from ...utils import logging
+from . import ReformerConfig, ReformerModelWithLMHead
 
 
 logging.set_verbosity_info()

src/transformers/models/roberta/convert_roberta_original_pytorch_checkpoint_to_pytorch.py

@@ -24,19 +24,9 @@ from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
 from fairseq.modules import TransformerSentenceEncoderLayer
 from packaging import version
 
-from transformers.models.bert.modeling_bert import (
-    BertIntermediate,
-    BertLayer,
-    BertOutput,
-    BertSelfAttention,
-    BertSelfOutput,
-)
-from transformers.models.roberta.modeling_roberta import (
-    RobertaConfig,
-    RobertaForMaskedLM,
-    RobertaForSequenceClassification,
-)
-from transformers.utils import logging
+from ...models.bert.modeling_bert import BertIntermediate, BertLayer, BertOutput, BertSelfAttention, BertSelfOutput
+from ...utils import logging
+from .modeling_roberta import RobertaConfig, RobertaForMaskedLM, RobertaForSequenceClassification
 
 
 if version.parse(fairseq.__version__) < version.parse("0.9.0"):

src/transformers/models/t5/convert_t5_original_tf_checkpoint_to_pytorch.py

@@ -17,8 +17,8 @@
 
 import argparse
 
-from transformers import T5Config, T5ForConditionalGeneration, load_tf_weights_in_t5
-from transformers.utils import logging
+from ...utils import logging
+from . import T5Config, T5ForConditionalGeneration, load_tf_weights_in_t5
 
 
 logging.set_verbosity_info()

src/transformers/models/t5/modeling_tf_t5.py

@@ -22,8 +22,6 @@ from typing import Tuple
 
 import tensorflow as tf
 
-from transformers.modeling_tf_utils import TFWrappedEmbeddings
-
 from ...activations_tf import get_tf_activation
 from ...file_utils import (
     DUMMY_INPUTS,
@@ -42,6 +40,7 @@ from ...modeling_tf_utils import (
     TFCausalLanguageModelingLoss,
     TFPreTrainedModel,
     TFSharedEmbeddings,
+    TFWrappedEmbeddings,
     input_processing,
     keras_serializable,
     shape_list,

src/transformers/models/tapas/convert_tapas_original_tf_checkpoint_to_pytorch.py

@@ -17,16 +17,16 @@
 
 import argparse
 
-from transformers.models.tapas.modeling_tapas import (
+from ...utils import logging
+from . import (
     TapasConfig,
     TapasForMaskedLM,
     TapasForQuestionAnswering,
     TapasForSequenceClassification,
     TapasModel,
+    TapasTokenizer,
     load_tf_weights_in_tapas,
 )
-from transformers.models.tapas.tokenization_tapas import TapasTokenizer
-from transformers.utils import logging
 
 
 logging.set_verbosity_info()

src/transformers/models/tapas/tokenization_tapas.py

@@ -28,9 +28,7 @@ from typing import Callable, Dict, Generator, List, Optional, Text, Tuple, Union
 
 import numpy as np
 
-from transformers import add_end_docstrings
-
-from ...file_utils import is_pandas_available
+from ...file_utils import add_end_docstrings, is_pandas_available
 from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
 from ...tokenization_utils_base import (
     ENCODE_KWARGS_DOCSTRING,

src/transformers/models/transfo_xl/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py

@@ -22,16 +22,11 @@ import sys
 
 import torch
 
-import transformers.models.transfo_xl.tokenization_transfo_xl as data_utils
-from transformers import (
-    CONFIG_NAME,
-    WEIGHTS_NAME,
-    TransfoXLConfig,
-    TransfoXLLMHeadModel,
-    load_tf_weights_in_transfo_xl,
-)
-from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
-from transformers.utils import logging
+from ...file_utils import CONFIG_NAME, WEIGHTS_NAME
+from ...utils import logging
+from . import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
+from . import tokenization_transfo_xl as data_utils
+from .tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
 
 
 logging.set_verbosity_info()

src/transformers/models/xlm/convert_xlm_original_pytorch_checkpoint_to_pytorch.py

@@ -21,9 +21,9 @@ import json
 import numpy
 import torch
 
-from transformers import CONFIG_NAME, WEIGHTS_NAME
-from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES
-from transformers.utils import logging
+from ...file_utils import CONFIG_NAME, WEIGHTS_NAME
+from ...utils import logging
+from .tokenization_xlm import VOCAB_FILES_NAMES
 
 
 logging.set_verbosity_info()

src/transformers/models/xlnet/convert_xlnet_original_tf_checkpoint_to_pytorch.py

@@ -20,16 +20,15 @@ import os
 
 import torch
 
-from transformers import (
-    CONFIG_NAME,
-    WEIGHTS_NAME,
+from ...file_utils import CONFIG_NAME, WEIGHTS_NAME
+from ...utils import logging
+from . import (
     XLNetConfig,
     XLNetForQuestionAnswering,
     XLNetForSequenceClassification,
     XLNetLMHeadModel,
     load_tf_weights_in_xlnet,
 )
-from transformers.utils import logging
 
 
 GLUE_TASKS_NUM_LABELS = {

templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/to_replace_{{cookiecutter.lowercase_modelname}}.py

@@ -24,71 +24,147 @@
 ##
 ## Put '## COMMENT' to comment on the file.
 
-
 # To replace in: "src/transformers/__init__.py"
-# Below: "if is_torch_available():" if generating PyTorch
+# Below: "    # PyTorch models structure" if generating PyTorch
 # Replace with:
 {% if cookiecutter.is_encoder_decoder_model == "False" %}
-    from .models.{{cookiecutter.lowercase_modelname}} import (
-        {{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST,
-        {{cookiecutter.camelcase_modelname}}ForMaskedLM,
-        {{cookiecutter.camelcase_modelname}}ForCausalLM,
-        {{cookiecutter.camelcase_modelname}}ForMultipleChoice,
-        {{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
-        {{cookiecutter.camelcase_modelname}}ForSequenceClassification,
-        {{cookiecutter.camelcase_modelname}}ForTokenClassification,
-        {{cookiecutter.camelcase_modelname}}Layer,
-        {{cookiecutter.camelcase_modelname}}Model,
-        {{cookiecutter.camelcase_modelname}}PreTrainedModel,
-        load_tf_weights_in_{{cookiecutter.lowercase_modelname}},
+    _import_structure["models.{{cookiecutter.lowercase_modelname}}"].extend(
+        [
+            "{{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST",
+            "{{cookiecutter.camelcase_modelname}}ForMaskedLM",
+            "{{cookiecutter.camelcase_modelname}}ForCausalLM",
+            "{{cookiecutter.camelcase_modelname}}ForMultipleChoice",
+            "{{cookiecutter.camelcase_modelname}}ForQuestionAnswering",
+            "{{cookiecutter.camelcase_modelname}}ForSequenceClassification",
+            "{{cookiecutter.camelcase_modelname}}ForTokenClassification",
+            "{{cookiecutter.camelcase_modelname}}Layer",
+            "{{cookiecutter.camelcase_modelname}}Model",
+            "{{cookiecutter.camelcase_modelname}}PreTrainedModel",
+            "load_tf_weights_in_{{cookiecutter.lowercase_modelname}}",
+        ]
     )
 {% else %}
-    from .models.{{cookiecutter.lowercase_modelname}} import (
-        {{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST,
-        {{cookiecutter.camelcase_modelname}}ForConditionalGeneration,
-        {{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
-        {{cookiecutter.camelcase_modelname}}ForSequenceClassification,
-        {{cookiecutter.camelcase_modelname}}Model,
+    _import_structure["models.{{cookiecutter.lowercase_modelname}}"].extend(
+        [
+            "{{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST",
+            "{{cookiecutter.camelcase_modelname}}ForConditionalGeneration",
+            "{{cookiecutter.camelcase_modelname}}ForQuestionAnswering",
+            "{{cookiecutter.camelcase_modelname}}ForSequenceClassification",
+            "{{cookiecutter.camelcase_modelname}}Model",
+        ]
     )
 {% endif -%}
 # End.
 
-# Below: "if is_tf_available():" if generating TensorFlow
+# Below: "    # TensorFlow models structure" if generating TensorFlow
 # Replace with:
 {% if cookiecutter.is_encoder_decoder_model == "False" %}
-    from .models.{{cookiecutter.lowercase_modelname}} import (
-        TF_{{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST,
-        TF{{cookiecutter.camelcase_modelname}}ForMaskedLM,
-        TF{{cookiecutter.camelcase_modelname}}ForCausalLM,
-        TF{{cookiecutter.camelcase_modelname}}ForMultipleChoice,
-        TF{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
-        TF{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
-        TF{{cookiecutter.camelcase_modelname}}ForTokenClassification,
-        TF{{cookiecutter.camelcase_modelname}}Layer,
-        TF{{cookiecutter.camelcase_modelname}}Model,
-        TF{{cookiecutter.camelcase_modelname}}PreTrainedModel,
+    _import_structure["models.{{cookiecutter.lowercase_modelname}}"].extend(
+        [
+            "TF_{{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST",
+            "TF{{cookiecutter.camelcase_modelname}}ForMaskedLM",
+            "TF{{cookiecutter.camelcase_modelname}}ForCausalLM",
+            "TF{{cookiecutter.camelcase_modelname}}ForMultipleChoice",
+            "TF{{cookiecutter.camelcase_modelname}}ForQuestionAnswering",
+            "TF{{cookiecutter.camelcase_modelname}}ForSequenceClassification",
+            "TF{{cookiecutter.camelcase_modelname}}ForTokenClassification",
+            "TF{{cookiecutter.camelcase_modelname}}Layer",
+            "TF{{cookiecutter.camelcase_modelname}}Model",
+            "TF{{cookiecutter.camelcase_modelname}}PreTrainedModel",
+        ]
     )
 {% else %}
-    from .models.{{cookiecutter.lowercase_modelname}} import (
-        TF{{cookiecutter.camelcase_modelname}}ForConditionalGeneration,
-        TF{{cookiecutter.camelcase_modelname}}Model,
-        TF{{cookiecutter.camelcase_modelname}}PreTrainedModel,
+    _import_structure["models.{{cookiecutter.lowercase_modelname}}"].extend(
+        [
+            "TF{{cookiecutter.camelcase_modelname}}ForConditionalGeneration",
+            "TF{{cookiecutter.camelcase_modelname}}Model",
+            "TF{{cookiecutter.camelcase_modelname}}PreTrainedModel",
+        ]
     )
 {% endif -%}
 # End.
 
-# Below: "if is_tokenizers_available():"
+# Below: "    # Fast tokenizers"
+# Replace with:
+    _import_structure["models.{{cookiecutter.lowercase_modelname}}"].append("{{cookiecutter.camelcase_modelname}}TokenizerFast")
+# End.
+
+# Below: "    # Models"
+# Replace with:
+    "models.{{cookiecutter.lowercase_modelname}}": ["{{cookiecutter.uppercase_modelname}}_PRETRAINED_CONFIG_ARCHIVE_MAP", "{{cookiecutter.camelcase_modelname}}Config", "{{cookiecutter.camelcase_modelname}}Tokenizer"],
+# End.
+
+# To replace in: "src/transformers/__init__.py"
+# Below: "    if is_torch_available():" if generating PyTorch
+# Replace with:
+{% if cookiecutter.is_encoder_decoder_model == "False" %}
+        from .models.{{cookiecutter.lowercase_modelname}} import (
+            {{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST,
+            {{cookiecutter.camelcase_modelname}}ForMaskedLM,
+            {{cookiecutter.camelcase_modelname}}ForCausalLM,
+            {{cookiecutter.camelcase_modelname}}ForMultipleChoice,
+            {{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
+            {{cookiecutter.camelcase_modelname}}ForSequenceClassification,
+            {{cookiecutter.camelcase_modelname}}ForTokenClassification,
+            {{cookiecutter.camelcase_modelname}}Layer,
+            {{cookiecutter.camelcase_modelname}}Model,
+            {{cookiecutter.camelcase_modelname}}PreTrainedModel,
+            load_tf_weights_in_{{cookiecutter.lowercase_modelname}},
+        )
+{% else %}
+        from .models.{{cookiecutter.lowercase_modelname}} import (
+            {{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST,
+            {{cookiecutter.camelcase_modelname}}ForConditionalGeneration,
+            {{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
+            {{cookiecutter.camelcase_modelname}}ForSequenceClassification,
+            {{cookiecutter.camelcase_modelname}}Model,
+        )
+{% endif -%}
+# End.
+
+# Below: "    if is_tf_available():" if generating TensorFlow
+# Replace with:
+{% if cookiecutter.is_encoder_decoder_model == "False" %}
+        from .models.{{cookiecutter.lowercase_modelname}} import (
+            TF_{{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST,
+            TF{{cookiecutter.camelcase_modelname}}ForMaskedLM,
+            TF{{cookiecutter.camelcase_modelname}}ForCausalLM,
+            TF{{cookiecutter.camelcase_modelname}}ForMultipleChoice,
+            TF{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
+            TF{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
+            TF{{cookiecutter.camelcase_modelname}}ForTokenClassification,
+            TF{{cookiecutter.camelcase_modelname}}Layer,
+            TF{{cookiecutter.camelcase_modelname}}Model,
+            TF{{cookiecutter.camelcase_modelname}}PreTrainedModel,
+        )
+{% else %}
+        from .models.{{cookiecutter.lowercase_modelname}} import (
+            TF{{cookiecutter.camelcase_modelname}}ForConditionalGeneration,
+            TF{{cookiecutter.camelcase_modelname}}Model,
+            TF{{cookiecutter.camelcase_modelname}}PreTrainedModel,
+        )
+{% endif -%}
+# End.
+
+# Below: "    if is_tokenizers_available():"
 # Replace with:
-    from .models.{{cookiecutter.lowercase_modelname}} import {{cookiecutter.camelcase_modelname}}TokenizerFast
+        from .models.{{cookiecutter.lowercase_modelname}} import {{cookiecutter.camelcase_modelname}}TokenizerFast
 # End.
 
-# Below: "from .models.albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig"
+# Below: "    from .models.albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig"
 # Replace with:
-from .models.{{cookiecutter.lowercase_modelname}} import {{cookiecutter.uppercase_modelname}}_PRETRAINED_CONFIG_ARCHIVE_MAP, {{cookiecutter.camelcase_modelname}}Config, {{cookiecutter.camelcase_modelname}}Tokenizer
+    from .models.{{cookiecutter.lowercase_modelname}} import {{cookiecutter.uppercase_modelname}}_PRETRAINED_CONFIG_ARCHIVE_MAP, {{cookiecutter.camelcase_modelname}}Config, {{cookiecutter.camelcase_modelname}}Tokenizer
 # End.
 
 
 
+# To replace in: "src/transformers/models/__init__.py"
+# Below: "from . import ("
+# Replace with:
+    {{cookiecutter.lowercase_modelname}},
+# End.
+
+
 # To replace in: "src/transformers/models/auto/configuration_auto.py"
 # Below: "# Add configs here"
 # Replace with:

utils/check_dummies.py

@@ -23,237 +23,79 @@ import re
 PATH_TO_TRANSFORMERS = "src/transformers"
 
 _re_single_line_import = re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")
+_re_test_backend = re.compile(r"^\s+if\s+is\_([a-z]*)\_available\(\):\s*$")
 
-DUMMY_CONSTANT = """
-{0} = None
-"""
-
-DUMMY_PT_PRETRAINED_CLASS = """
-class {0}:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-    @classmethod
-    def from_pretrained(self, *args, **kwargs):
-        requires_pytorch(self)
-"""
-
-DUMMY_PT_CLASS = """
-class {0}:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-"""
 
-DUMMY_PT_FUNCTION = """
-def {0}(*args, **kwargs):
-    requires_pytorch({0})
-"""
+BACKENDS = ["torch", "tf", "flax", "sentencepiece", "tokenizers"]
 
 
-DUMMY_TF_PRETRAINED_CLASS = """
-class {0}:
-    def __init__(self, *args, **kwargs):
-        requires_tf(self)
-
-    @classmethod
-    def from_pretrained(self, *args, **kwargs):
-        requires_tf(self)
-"""
-
-DUMMY_TF_CLASS = """
-class {0}:
-    def __init__(self, *args, **kwargs):
-        requires_tf(self)
-"""
-
-DUMMY_TF_FUNCTION = """
-def {0}(*args, **kwargs):
-    requires_tf({0})
-"""
-
-
-DUMMY_FLAX_PRETRAINED_CLASS = """
-class {0}:
-    def __init__(self, *args, **kwargs):
-        requires_flax(self)
-
-    @classmethod
-    def from_pretrained(self, *args, **kwargs):
-        requires_flax(self)
-"""
-
-DUMMY_FLAX_CLASS = """
-class {0}:
-    def __init__(self, *args, **kwargs):
-        requires_flax(self)
-"""
-
-DUMMY_FLAX_FUNCTION = """
-def {0}(*args, **kwargs):
-    requires_flax({0})
-"""
-
-
-DUMMY_SENTENCEPIECE_PRETRAINED_CLASS = """
-class {0}:
-    def __init__(self, *args, **kwargs):
-        requires_sentencepiece(self)
-
-    @classmethod
-    def from_pretrained(self, *args, **kwargs):
-        requires_sentencepiece(self)
-"""
-
-DUMMY_SENTENCEPIECE_CLASS = """
-class {0}:
-    def __init__(self, *args, **kwargs):
-        requires_sentencepiece(self)
-"""
-
-DUMMY_SENTENCEPIECE_FUNCTION = """
-def {0}(*args, **kwargs):
-    requires_sentencepiece({0})
+DUMMY_CONSTANT = """
+{0} = None
 """
 
-
-DUMMY_TOKENIZERS_PRETRAINED_CLASS = """
+DUMMY_PRETRAINED_CLASS = """
 class {0}:
     def __init__(self, *args, **kwargs):
-        requires_tokenizers(self)
+        requires_{1}(self)
 
     @classmethod
     def from_pretrained(self, *args, **kwargs):
-        requires_tokenizers(self)
+        requires_{1}(self)
 """
 
-DUMMY_TOKENIZERS_CLASS = """
+DUMMY_CLASS = """
 class {0}:
     def __init__(self, *args, **kwargs):
-        requires_tokenizers(self)
+        requires_{1}(self)
 """
 
-DUMMY_TOKENIZERS_FUNCTION = """
+DUMMY_FUNCTION = """
 def {0}(*args, **kwargs):
-    requires_tokenizers({0})
+    requires_{1}({0})
 """
 
-# Map all these to dummy type
-
-DUMMY_PRETRAINED_CLASS = {
-    "pt": DUMMY_PT_PRETRAINED_CLASS,
-    "tf": DUMMY_TF_PRETRAINED_CLASS,
-    "flax": DUMMY_FLAX_PRETRAINED_CLASS,
-    "sentencepiece": DUMMY_SENTENCEPIECE_PRETRAINED_CLASS,
-    "tokenizers": DUMMY_TOKENIZERS_PRETRAINED_CLASS,
-}
-
-DUMMY_CLASS = {
-    "pt": DUMMY_PT_CLASS,
-    "tf": DUMMY_TF_CLASS,
-    "flax": DUMMY_FLAX_CLASS,
-    "sentencepiece": DUMMY_SENTENCEPIECE_CLASS,
-    "tokenizers": DUMMY_TOKENIZERS_CLASS,
-}
-
-DUMMY_FUNCTION = {
-    "pt": DUMMY_PT_FUNCTION,
-    "tf": DUMMY_TF_FUNCTION,
-    "flax": DUMMY_FLAX_FUNCTION,
-    "sentencepiece": DUMMY_SENTENCEPIECE_FUNCTION,
-    "tokenizers": DUMMY_TOKENIZERS_FUNCTION,
-}
-
 
 def read_init():
     """ Read the init and extracts PyTorch, TensorFlow, SentencePiece and Tokenizers objects. """
     with open(os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"), "r", encoding="utf-8", newline="\n") as f:
         lines = f.readlines()
 
+    # Get to the point we do the actual imports for type checking
     line_index = 0
-    # Find where the SentencePiece imports begin
-    sentencepiece_objects = []
-    while not lines[line_index].startswith("if is_sentencepiece_available():"):
-        line_index += 1
-    line_index += 1
-
-    # Until we unindent, add SentencePiece objects to the list
-    while len(lines[line_index]) <= 1 or lines[line_index].startswith("    "):
-        line = lines[line_index]
-        search = _re_single_line_import.search(line)
-        if search is not None:
-            sentencepiece_objects += search.groups()[0].split(", ")
-        elif line.startswith("        "):
-            sentencepiece_objects.append(line[8:-2])
-        line_index += 1
-
-    # Find where the Tokenizers imports begin
-    tokenizers_objects = []
-    while not lines[line_index].startswith("if is_tokenizers_available():"):
-        line_index += 1
-    line_index += 1
-
-    # Until we unindent, add Tokenizers objects to the list
-    while len(lines[line_index]) <= 1 or lines[line_index].startswith("    "):
-        line = lines[line_index]
-        search = _re_single_line_import.search(line)
-        if search is not None:
-            tokenizers_objects += search.groups()[0].split(", ")
-        elif line.startswith("        "):
-            tokenizers_objects.append(line[8:-2])
-        line_index += 1
-
-    # Find where the PyTorch imports begin
-    pt_objects = []
-    while not lines[line_index].startswith("if is_torch_available():"):
-        line_index += 1
-    line_index += 1
-
-    # Until we unindent, add PyTorch objects to the list
-    while len(lines[line_index]) <= 1 or lines[line_index].startswith("    "):
-        line = lines[line_index]
-        search = _re_single_line_import.search(line)
-        if search is not None:
-            pt_objects += search.groups()[0].split(", ")
-        elif line.startswith("        "):
-            pt_objects.append(line[8:-2])
+    while not lines[line_index].startswith("if TYPE_CHECKING"):
         line_index += 1
 
-    # Find where the TF imports begin
-    tf_objects = []
-    while not lines[line_index].startswith("if is_tf_available():"):
-        line_index += 1
-    line_index += 1
-
-    # Until we unindent, add PyTorch objects to the list
-    while len(lines[line_index]) <= 1 or lines[line_index].startswith("    "):
-        line = lines[line_index]
-        search = _re_single_line_import.search(line)
-        if search is not None:
-            tf_objects += search.groups()[0].split(", ")
-        elif line.startswith("        "):
-            tf_objects.append(line[8:-2])
-        line_index += 1
-
-    # Find where the FLAX imports begin
-    flax_objects = []
-    while not lines[line_index].startswith("if is_flax_available():"):
-        line_index += 1
-    line_index += 1
-
-    # Until we unindent, add PyTorch objects to the list
-    while len(lines[line_index]) <= 1 or lines[line_index].startswith("    "):
-        line = lines[line_index]
-        search = _re_single_line_import.search(line)
-        if search is not None:
-            flax_objects += search.groups()[0].split(", ")
-        elif line.startswith("        "):
-            flax_objects.append(line[8:-2])
-        line_index += 1
+    backend_specific_objects = {}
+    # Go through the end of the file
+    while line_index < len(lines):
+        # If the line is an if is_backemd_available, we grab all objects associated.
+        if _re_test_backend.search(lines[line_index]) is not None:
+            backend = _re_test_backend.search(lines[line_index]).groups()[0]
+            line_index += 1
+
+            # Ignore if backend isn't tracked for dummies.
+            if backend not in BACKENDS:
+                continue
+
+            objects = []
+            # Until we unindent, add backend objects to the list
+            while len(lines[line_index]) <= 1 or lines[line_index].startswith(" " * 8):
+                line = lines[line_index]
+                single_line_import_search = _re_single_line_import.search(line)
+                if single_line_import_search is not None:
+                    objects.extend(single_line_import_search.groups()[0].split(", "))
+                elif line.startswith(" " * 12):
+                    objects.append(line[12:-2])
+                line_index += 1
+
+            backend_specific_objects[backend] = objects
+        else:
+            line_index += 1
 
-    return sentencepiece_objects, tokenizers_objects, pt_objects, tf_objects, flax_objects
+    return backend_specific_objects
 
 
-def create_dummy_object(name, type="pt"):
+def create_dummy_object(name, backend_name):
     """ Create the code for the dummy object corresponding to `name`."""
     _pretrained = [
         "Config" "ForCausalLM",
@@ -266,11 +108,10 @@ def create_dummy_object(name, type="pt"):
         "Model",
         "Tokenizer",
     ]
-    assert type in ["pt", "tf", "sentencepiece", "tokenizers", "flax"]
     if name.isupper():
         return DUMMY_CONSTANT.format(name)
     elif name.islower():
-        return (DUMMY_FUNCTION[type]).format(name)
+        return DUMMY_FUNCTION.format(name, backend_name)
     else:
         is_pretrained = False
         for part in _pretrained:
@@ -278,114 +119,61 @@ def create_dummy_object(name, type="pt"):
                 is_pretrained = True
                 break
         if is_pretrained:
-            template = DUMMY_PRETRAINED_CLASS[type]
+            return DUMMY_PRETRAINED_CLASS.format(name, backend_name)
         else:
-            template = DUMMY_CLASS[type]
-        return template.format(name)
+            return DUMMY_CLASS.format(name, backend_name)
 
 
 def create_dummy_files():
     """ Create the content of the dummy files. """
-    sentencepiece_objects, tokenizers_objects, pt_objects, tf_objects, flax_objects = read_init()
+    backend_specific_objects = read_init()
+    # For special correspondence backend to module name as used in the function requires_modulename
+    module_names = {"torch": "pytorch"}
+    dummy_files = {}
 
-    sentencepiece_dummies = "# This file is autogenerated by the command `make fix-copies`, do not edit.\n"
-    sentencepiece_dummies += "from ..file_utils import requires_sentencepiece\n\n"
-    sentencepiece_dummies += "\n".join([create_dummy_object(o, type="sentencepiece") for o in sentencepiece_objects])
+    for backend, objects in backend_specific_objects.items():
+        backend_name = module_names.get(backend, backend)
+        dummy_file = "# This file is autogenerated by the command `make fix-copies`, do not edit.\n"
+        dummy_file += f"from ..file_utils import requires_{backend_name}\n\n"
+        dummy_file += "\n".join([create_dummy_object(o, backend_name) for o in objects])
+        dummy_files[backend] = dummy_file
 
-    tokenizers_dummies = "# This file is autogenerated by the command `make fix-copies`, do not edit.\n"
-    tokenizers_dummies += "from ..file_utils import requires_tokenizers\n\n"
-    tokenizers_dummies += "\n".join([create_dummy_object(o, type="tokenizers") for o in tokenizers_objects])
-
-    pt_dummies = "# This file is autogenerated by the command `make fix-copies`, do not edit.\n"
-    pt_dummies += "from ..file_utils import requires_pytorch\n\n"
-    pt_dummies += "\n".join([create_dummy_object(o, type="pt") for o in pt_objects])
-
-    tf_dummies = "# This file is autogenerated by the command `make fix-copies`, do not edit.\n"
-    tf_dummies += "from ..file_utils import requires_tf\n\n"
-    tf_dummies += "\n".join([create_dummy_object(o, type="tf") for o in tf_objects])
-
-    flax_dummies = "# This file is autogenerated by the command `make fix-copies`, do not edit.\n"
-    flax_dummies += "from ..file_utils import requires_flax\n\n"
-    flax_dummies += "\n".join([create_dummy_object(o, type="flax") for o in flax_objects])
-
-    return sentencepiece_dummies, tokenizers_dummies, pt_dummies, tf_dummies, flax_dummies
+    return dummy_files
 
 
 def check_dummies(overwrite=False):
     """ Check if the dummy files are up to date and maybe `overwrite` with the right content. """
-    sentencepiece_dummies, tokenizers_dummies, pt_dummies, tf_dummies, flax_dummies = create_dummy_files()
+    dummy_files = create_dummy_files()
+    # For special correspondence backend to shortcut as used in utils/dummy_xxx_objects.py
+    short_names = {"torch": "pt"}
+
+    # Locate actual dummy modules and read their content.
     path = os.path.join(PATH_TO_TRANSFORMERS, "utils")
-    sentencepiece_file = os.path.join(path, "dummy_sentencepiece_objects.py")
-    tokenizers_file = os.path.join(path, "dummy_tokenizers_objects.py")
-    pt_file = os.path.join(path, "dummy_pt_objects.py")
-    tf_file = os.path.join(path, "dummy_tf_objects.py")
-    flax_file = os.path.join(path, "dummy_flax_objects.py")
-
-    with open(sentencepiece_file, "r", encoding="utf-8", newline="\n") as f:
-        actual_sentencepiece_dummies = f.read()
-    with open(tokenizers_file, "r", encoding="utf-8", newline="\n") as f:
-        actual_tokenizers_dummies = f.read()
-    with open(pt_file, "r", encoding="utf-8", newline="\n") as f:
-        actual_pt_dummies = f.read()
-    with open(tf_file, "r", encoding="utf-8", newline="\n") as f:
-        actual_tf_dummies = f.read()
-    with open(flax_file, "r", encoding="utf-8", newline="\n") as f:
-        actual_flax_dummies = f.read()
-
-    if sentencepiece_dummies != actual_sentencepiece_dummies:
-        if overwrite:
-            print("Updating transformers.utils.dummy_sentencepiece_objects.py as the main __init__ has new objects.")
-            with open(sentencepiece_file, "w", encoding="utf-8", newline="\n") as f:
-                f.write(sentencepiece_dummies)
-        else:
-            raise ValueError(
-                "The main __init__ has objects that are not present in transformers.utils.dummy_sentencepiece_objects.py.",
-                "Run `make fix-copies` to fix this.",
-            )
-
-    if tokenizers_dummies != actual_tokenizers_dummies:
-        if overwrite:
-            print("Updating transformers.utils.dummy_tokenizers_objects.py as the main __init__ has new objects.")
-            with open(tokenizers_file, "w", encoding="utf-8", newline="\n") as f:
-                f.write(tokenizers_dummies)
-        else:
-            raise ValueError(
-                "The main __init__ has objects that are not present in transformers.utils.dummy_tokenizers_objects.py.",
-                "Run `make fix-copies` to fix this.",
-            )
-
-    if pt_dummies != actual_pt_dummies:
-        if overwrite:
-            print("Updating transformers.utils.dummy_pt_objects.py as the main __init__ has new objects.")
-            with open(pt_file, "w", encoding="utf-8", newline="\n") as f:
-                f.write(pt_dummies)
-        else:
-            raise ValueError(
-                "The main __init__ has objects that are not present in transformers.utils.dummy_pt_objects.py.",
-                "Run `make fix-copies` to fix this.",
-            )
-
-    if tf_dummies != actual_tf_dummies:
-        if overwrite:
-            print("Updating transformers.utils.dummy_tf_objects.py as the main __init__ has new objects.")
-            with open(tf_file, "w", encoding="utf-8", newline="\n") as f:
-                f.write(tf_dummies)
-        else:
-            raise ValueError(
-                "The main __init__ has objects that are not present in transformers.utils.dummy_pt_objects.py.",
-                "Run `make fix-copies` to fix this.",
-            )
-
-    if flax_dummies != actual_flax_dummies:
-        if overwrite:
-            print("Updating transformers.utils.dummy_flax_objects.py as the main __init__ has new objects.")
-            with open(flax_file, "w", encoding="utf-8", newline="\n") as f:
-                f.write(flax_dummies)
-        else:
-            raise ValueError(
-                "The main __init__ has objects that are not present in transformers.utils.dummy_flax_objects.py.",
-                "Run `make fix-copies` to fix this.",
-            )
+    dummy_file_paths = {
+        backend: os.path.join(path, f"dummy_{short_names.get(backend, backend)}_objects.py")
+        for backend in dummy_files.keys()
+    }
+
+    actual_dummies = {}
+    for backend, file_path in dummy_file_paths.items():
+        with open(file_path, "r", encoding="utf-8", newline="\n") as f:
+            actual_dummies[backend] = f.read()
+
+    for backend in dummy_files.keys():
+        if dummy_files[backend] != actual_dummies[backend]:
+            if overwrite:
+                print(
+                    f"Updating transformers.utils.dummy_{short_names.get(backend, backend)}_objects.py as the main "
+                    "__init__ has new objects."
+                )
+                with open(dummy_file_paths[backend], "w", encoding="utf-8", newline="\n") as f:
+                    f.write(dummy_files[backend])
+            else:
+                raise ValueError(
+                    "The main __init__ has objects that are not present in "
+                    f"transformers.utils.dummy_{short_names.get(backend, backend)}_objects.py. Run `make fix-copies` "
+                    "to fix this."
+                )
 
 
 if __name__ == "__main__":

utils/check_repo.py

@@ -413,9 +413,6 @@ SHOULD_HAVE_THEIR_OWN_PAGE = [
 def ignore_undocumented(name):
     """Rules to determine if `name` should be undocumented."""
     # NOT DOCUMENTED ON PURPOSE.
-    # Magic attributes are not documented.
-    if name.startswith("__"):
-        return True
     # Constants uppercase are not documented.
     if name.isupper():
         return True
@@ -459,7 +456,9 @@ def ignore_undocumented(name):
 def check_all_objects_are_documented():
     """ Check all models are properly documented."""
     documented_objs = find_all_documented_objects()
-    undocumented_objs = [c for c in dir(transformers) if c not in documented_objs and not ignore_undocumented(c)]
+    modules = transformers._modules
+    objects = [c for c in dir(transformers) if c not in modules and not c.startswith("_")]
+    undocumented_objs = [c for c in objects if c not in documented_objs and not ignore_undocumented(c)]
     if len(undocumented_objs) > 0:
         raise Exception(
             "The following objects are in the public init so should be documented:\n - "