Use real tokenizers if tiny version(s) creation has issue(s) (#22428)

Fix some tiny model creation issues
Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

.github/workflows/update_tiny_models.yml

-name: Self-hosted runner (push)
+name: Update Tiny Models
 
 on:
   push:

tests/models/bigbird_pegasus/test_modeling_bigbird_pegasus.py

@@ -268,6 +268,15 @@ class BigBirdPegasusModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineT
     # Also torchscript is not an important feature to have in the beginning.
     test_torchscript = False
 
+    # TODO: Fix the failed tests
+    def is_pipeline_test_to_skip(
+        self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name
+    ):
+        if pipeline_test_casse_name == "QAPipelineTests" and not tokenizer_name.endswith("Fast"):
+            return True
+
+        return False
+
     # overwrite from GenerationTesterMixin to solve problem
     # with conflicting random seeds
     def _get_input_ids_and_config(self):

tests/models/xlm_roberta_xl/test_modeling_xlm_roberta_xl.py

@@ -387,6 +387,15 @@ class XLMRobertaXLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTes
         else {}
     )
 
+    # TODO: Fix the failed tests
+    def is_pipeline_test_to_skip(
+        self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name
+    ):
+        if pipeline_test_casse_name == "QAPipelineTests" and not tokenizer_name.endswith("Fast"):
+            return True
+
+        return False
+
     def setUp(self):
         self.model_tester = XLMRobertaXLModelTester(self)
         self.config_tester = ConfigTester(self, config_class=XLMRobertaXLConfig, hidden_size=37)

tests/models/xmod/test_modeling_xmod.py

@@ -384,6 +384,15 @@ class XmodModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin
         else {}
     )
 
+    # TODO: Fix the failed tests
+    def is_pipeline_test_to_skip(
+        self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name
+    ):
+        if pipeline_test_casse_name == "QAPipelineTests" and not tokenizer_name.endswith("Fast"):
+            return True
+
+        return False
+
     def setUp(self):
         self.model_tester = XmodModelTester(self)
         self.config_tester = ConfigTester(self, config_class=XmodConfig, hidden_size=37)

utils/create_dummy_models.py

@@ -405,7 +405,11 @@ def get_tiny_config(config_class, model_class=None, **model_tester_kwargs):
             for _tester_classes in models_to_model_testers.values():
                 tester_classes.extend(_tester_classes)
         if len(tester_classes) > 0:
-            model_tester_class = sorted(tester_classes, key=lambda x: x.__name__)[0]
+            # sort with the length of the class names first, then the alphabetical order
+            # This is to avoid `T5EncoderOnlyModelTest` is used instead of `T5ModelTest`, which has
+            # `is_encoder_decoder=False` and causes some pipeline tests failing (also failures in `Optimum` CI).
+            # TODO: More fine grained control of the desired tester class.
+            model_tester_class = sorted(tester_classes, key=lambda x: (len(x.__name__), x.__name__))[0]
     except ModuleNotFoundError:
         error = f"Tiny config not created for {model_type} - cannot find the testing module from the model name."
         raise ValueError(error)
@@ -484,22 +488,68 @@ def convert_processors(processors, tiny_config, output_folder, result):
     This method should not fail: we catch the errors and put them in `result["warnings"]` with descriptive messages.
     """
 
+    def _sanity_check(fast_tokenizer, slow_tokenizer, keep_fast_tokenizer=False):
+        """Set tokenizer(s) to `None` if the fast/slow tokenizers have different values for `vocab_size` or `length`.
+
+        If `keep_fast_tokenizer=True`, the fast tokenizer will be kept.
+        """
+        # sanity check 1: fast and slow tokenizers should be compatible (vocab_size)
+        if fast_tokenizer is not None and slow_tokenizer is not None:
+            if fast_tokenizer.vocab_size != slow_tokenizer.vocab_size:
+                warning_messagae = (
+                    "The fast/slow tokenizers "
+                    f"({fast_tokenizer.__class__.__name__}/{slow_tokenizer.__class__.__name__}) have different "
+                    "vocabulary size: "
+                    f"fast_tokenizer.vocab_size = {fast_tokenizer.vocab_size} and "
+                    f"slow_tokenizer.vocab_size = {slow_tokenizer.vocab_size}."
+                )
+                result["warnings"].append(warning_messagae)
+                if not keep_fast_tokenizer:
+                    fast_tokenizer = None
+                slow_tokenizer = None
+
+        # sanity check 2: fast and slow tokenizers should be compatible (length)
+        if fast_tokenizer is not None and slow_tokenizer is not None:
+            if len(fast_tokenizer) != len(slow_tokenizer):
+                warning_messagae = (
+                    f"The fast/slow tokenizers () have different length: "
+                    f"len(fast_tokenizer) = {len(fast_tokenizer)} and "
+                    f"len(slow_tokenizer) = {len(slow_tokenizer)}."
+                )
+                result["warnings"].append(warning_messagae)
+                if not keep_fast_tokenizer:
+                    fast_tokenizer = None
+                slow_tokenizer = None
+
+        return fast_tokenizer, slow_tokenizer
+
     tokenizers = []
     feature_extractors = []
     for processor in processors:
         if isinstance(processor, PreTrainedTokenizerBase):
-            tokenizers.append(processor)
+            if processor.__class__.__name__ not in {x.__class__.__name__ for x in tokenizers}:
+                tokenizers.append(processor)
         elif isinstance(processor, BaseImageProcessor):
-            feature_extractors.append(processor)
+            if processor.__class__.__name__ not in {x.__class__.__name__ for x in feature_extractors}:
+                feature_extractors.append(processor)
         elif isinstance(processor, FeatureExtractionMixin):
-            feature_extractors.append(processor)
+            if processor.__class__.__name__ not in {x.__class__.__name__ for x in feature_extractors}:
+                feature_extractors.append(processor)
         elif isinstance(processor, ProcessorMixin):
+            if hasattr(processor, "tokenizer"):
+                if processor.tokenizer.__class__.__name__ not in {x.__class__.__name__ for x in tokenizers}:
+                    tokenizers.append(processor.tokenizer)
             # Currently, we only have these 2 possibilities
-            tokenizers.append(processor.tokenizer)
             if hasattr(processor, "image_processor"):
-                feature_extractors.append(processor.image_processor)
+                if processor.image_processor.__class__.__name__ not in {
+                    x.__class__.__name__ for x in feature_extractors
+                }:
+                    feature_extractors.append(processor.image_processor)
             elif hasattr(processor, "feature_extractor"):
-                feature_extractors.append(processor.feature_extractor)
+                if processor.feature_extractor.__class__.__name__ not in {
+                    x.__class__.__name__ for x in feature_extractors
+                }:
+                    feature_extractors.append(processor.feature_extractor)
 
     # check the built processors have the unique type
     num_types = len({x.__class__.__name__ for x in feature_extractors})
@@ -511,65 +561,109 @@ def convert_processors(processors, tiny_config, output_folder, result):
 
     fast_tokenizer = None
     slow_tokenizer = None
+
     for tokenizer in tokenizers:
         if isinstance(tokenizer, PreTrainedTokenizerFast):
-            if fast_tokenizer is None:
-                fast_tokenizer = tokenizer
-                try:
-                    # Wav2Vec2ForCTC , ByT5Tokenizer etc. all are already small enough and have no fast version that can
-                    # be retrained
-                    if fast_tokenizer.vocab_size > TARGET_VOCAB_SIZE:
-                        fast_tokenizer = convert_tokenizer(tokenizer)
-                except Exception:
-                    result["warnings"].append(
-                        (
-                            f"Failed to convert the fast tokenizer for {fast_tokenizer.__class__.__name__}.",
-                            traceback.format_exc(),
-                        )
-                    )
-                    continue
-        elif slow_tokenizer is None:
+            fast_tokenizer = tokenizer
+        else:
             slow_tokenizer = tokenizer
 
-    # Make sure the fast tokenizer can be saved
+    # If the (original) fast/slow tokenizers don't correspond, keep only the fast tokenizer.
+    # This doesn't necessarily imply the fast/slow tokenizers in a single Hub repo. has issues.
+    # It's more of an issue in `build_processor` which tries to get a checkpoint with as much effort as possible.
+    # For `YosoModel` (which uses `AlbertTokenizer(Fast)`), its real (Hub) checkpoint doesn't contain valid files to
+    # load the slower tokenizer (`AlbertTokenizer`), and it ends up finding the (canonical) checkpoint of `AlbertModel`,
+    # which has different vocabulary.
+    # TODO: Try to improve `build_processor`'s definition and/or usage to avoid the above situation in the first place.
+    fast_tokenizer, slow_tokenizer = _sanity_check(fast_tokenizer, slow_tokenizer, keep_fast_tokenizer=True)
+    original_fast_tokenizer, original_slow_tokenizer = fast_tokenizer, slow_tokenizer
+
     if fast_tokenizer:
         try:
-            fast_tokenizer.save_pretrained(output_folder)
+            # Wav2Vec2ForCTC , ByT5Tokenizer etc. all are already small enough and have no fast version that can
+            # be retrained
+            if fast_tokenizer.vocab_size > TARGET_VOCAB_SIZE:
+                fast_tokenizer = convert_tokenizer(fast_tokenizer)
         except Exception:
             result["warnings"].append(
                 (
-                    f"Failed to save the fast tokenizer for {fast_tokenizer.__class__.__name__}.",
+                    f"Failed to convert the fast tokenizer for {fast_tokenizer.__class__.__name__}.",
                     traceback.format_exc(),
                 )
             )
-            fast_tokenizer = None
 
-    # Make sure the slow tokenizer (if any) corresponds to the fast version (as it might be converted above)
+    # If `fast_tokenizer` exists, `slow_tokenizer` should correspond to it.
     if fast_tokenizer:
+        # Make sure the fast tokenizer can be saved
         try:
-            slow_tokenizer = AutoTokenizer.from_pretrained(output_folder, use_fast=False)
+            # We don't save it to `output_folder` at this moment - only at the end of this function.
+            with tempfile.TemporaryDirectory() as tmpdir:
+                fast_tokenizer.save_pretrained(tmpdir)
+                try:
+                    slow_tokenizer = AutoTokenizer.from_pretrained(tmpdir, use_fast=False)
+                except Exception:
+                    result["warnings"].append(
+                        (
+                            f"Failed to load the slow tokenizer saved from {fast_tokenizer.__class__.__name__}.",
+                            traceback.format_exc(),
+                        )
+                    )
+                    # Let's just keep the fast version
+                    slow_tokenizer = None
         except Exception:
             result["warnings"].append(
                 (
-                    f"Failed to load the slow tokenizer saved from {fast_tokenizer.__class__.__name__}.",
+                    f"Failed to save the fast tokenizer for {fast_tokenizer.__class__.__name__}.",
                     traceback.format_exc(),
                 )
             )
-            # Let's just keep the fast version
-            slow_tokenizer = None
+            fast_tokenizer = None
 
-    # If the fast version can't be created and saved, let's use the slow version
-    if not fast_tokenizer and slow_tokenizer:
-        try:
-            slow_tokenizer.save_pretrained(output_folder)
-        except Exception:
-            result["warnings"].append(
-                (
-                    f"Failed to save the slow tokenizer for {slow_tokenizer.__class__.__name__}.",
-                    traceback.format_exc(),
+    # If the (possibly converted) fast/slow tokenizers don't correspond, set them to `None`, and use the original
+    # tokenizers.
+    fast_tokenizer, slow_tokenizer = _sanity_check(fast_tokenizer, slow_tokenizer, keep_fast_tokenizer=False)
+
+    # If there is any conversion failed, we keep the original tokenizers.
+    if (original_fast_tokenizer is not None and fast_tokenizer is None) or (
+        original_slow_tokenizer is not None and slow_tokenizer is None
+    ):
+        warning_messagae = (
+            "There are some issues when converting the fast/slow tokenizers. The original tokenizers from the Hub "
+            " will be used instead."
+        )
+        result["warnings"].append(warning_messagae)
+        # Let's use the original version at the end (`original_fast_tokenizer` and `original_slow_tokenizer`)
+        fast_tokenizer = original_fast_tokenizer
+        slow_tokenizer = original_slow_tokenizer
+
+    # Make sure the fast tokenizer can be saved
+    if fast_tokenizer:
+        # We don't save it to `output_folder` at this moment - only at the end of this function.
+        with tempfile.TemporaryDirectory() as tmpdir:
+            try:
+                fast_tokenizer.save_pretrained(tmpdir)
+            except Exception:
+                result["warnings"].append(
+                    (
+                        f"Failed to save the fast tokenizer for {fast_tokenizer.__class__.__name__}.",
+                        traceback.format_exc(),
+                    )
                 )
-            )
-            slow_tokenizer = None
+                fast_tokenizer = None
+    # Make sure the slow tokenizer can be saved
+    if slow_tokenizer:
+        # We don't save it to `output_folder` at this moment - only at the end of this function.
+        with tempfile.TemporaryDirectory() as tmpdir:
+            try:
+                slow_tokenizer.save_pretrained(tmpdir)
+            except Exception:
+                result["warnings"].append(
+                    (
+                        f"Failed to save the slow tokenizer for {slow_tokenizer.__class__.__name__}.",
+                        traceback.format_exc(),
+                    )
+                )
+                slow_tokenizer = None
 
     # update feature extractors using the tiny config
     try:
@@ -883,7 +977,9 @@ def get_config_overrides(config_class, processors):
         return config_overrides
 
     # Get some properties of the (already converted) tokenizer (smaller vocab size, special token ids, etc.)
-    vocab_size = tokenizer.vocab_size
+    # We use `len(tokenizer)` instead of `tokenizer.vocab_size` to avoid potential issues for tokenizers with non-empty
+    # `added_tokens_encoder`. One example is the `DebertaV2Tokenizer` where the mask token is the extra token.
+    vocab_size = len(tokenizer)
     config_overrides["vocab_size"] = vocab_size
 
     # Used to create a new model tester with `tokenizer.vocab_size` in order to get the (updated) special token ids.