Add possibility to ignore imports in test_fecther (#12801)

* Add possibility to ignore imports in test_fecther

* Style

src/transformers/tokenization_utils_base.py

@@ -1769,7 +1769,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
             init_kwargs = init_configuration
 
         if config_tokenizer_class is None:
-            from .models.auto.configuration_auto import AutoConfig
+            from .models.auto.configuration_auto import AutoConfig  # tests_ignore
 
             # Second attempt. If we have not yet found tokenizer_class, let's try to use the config.
             try:
@@ -1781,8 +1781,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
             if config_tokenizer_class is None:
                 # Third attempt. If we have not yet found the original type of the tokenizer,
                 # we are loading we see if we can infer it from the type of the configuration file
-                from .models.auto.configuration_auto import CONFIG_MAPPING
-                from .models.auto.tokenization_auto import TOKENIZER_MAPPING
+                from .models.auto.configuration_auto import CONFIG_MAPPING  # tests_ignore
+                from .models.auto.tokenization_auto import TOKENIZER_MAPPING  # tests_ignore
 
                 if hasattr(config, "model_type"):
                     config_class = CONFIG_MAPPING.get(config.model_type)

utils/tests_fetcher.py

@@ -131,7 +131,8 @@ def get_module_dependencies(module_fname):
     imported_modules = []
 
     # Let's start with relative imports
-    relative_imports = re.findall(r"from\s+(\.+\S+)\s+import\s+\S+\s", content)
+    relative_imports = re.findall(r"from\s+(\.+\S+)\s+import\s+([^\n]+)\n", content)
+    relative_imports = [mod for mod, imp in relative_imports if "# tests_ignore" not in imp]
     for imp in relative_imports:
         level = 0
         while imp.startswith("."):
@@ -151,7 +152,8 @@ def get_module_dependencies(module_fname):
     # Let's continue with direct imports
     # The import from the transformers module are ignored for the same reason we ignored the
     # main init before.
-    direct_imports = re.findall(r"from\s+transformers\.(\S+)\s+import\s+\S+\s", content)
+    direct_imports = re.findall(r"from\s+transformers\.(\S+)\s+import\s+([^\n]+)\n", content)
+    direct_imports = [mod for mod, imp in direct_imports if "# tests_ignore" not in imp]
     for imp in direct_imports:
         import_parts = imp.split(".")
         dep_parts = ["src", "transformers"] + import_parts