Easily train a new fast tokenizer from a given one (#12361)

* [WIP] Easily train a new fast tokenizer from a given one

* Fix test

* Roll out to other tokenizers and add tests

* Fix bug with unk id and add emoji to test

* Really use something different in test

* Implement special tokens map

* Map special tokens in the Transformers tokenizers

* Fix test

* Make test more robust

* Fix test for BPE

* More robust map and test

Co-authored-by SaulLu

* Test file

* Stronger tests
Co-authored-by: SaulLu <lucilesaul.com@gmail.com>

* Map unk token for Wordpiece and address review comment

* Fix lowercase test and address review comment

* Fix all tests

* Simplify test

* Fix tests for realsies

* Easily train a new fast tokenizer from a given one - tackle the special tokens format (str or AddedToken) (#12420)

* Propose change in tests regarding lower case

* add new test for special tokens types

* put back the test part about decoding

* add feature: the AddedToken is re-build with the different mapped content

* Address review comment: simplify AddedToken building
Co-authored-by: sgugger <sylvain.gugger@gmail.com>

* Update src/transformers/tokenization_utils_fast.py
Co-authored-by: sgugger <sylvain.gugger@gmail.com>
Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
Co-authored-by: SaulLu <lucilesaul.com@gmail.com>
Co-authored-by: SaulLu <55560583+SaulLu@users.noreply.github.com>

src/transformers/models/albert/tokenization_albert_fast.py

@@ -121,7 +121,7 @@ class AlbertTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
+        vocab_file=None,
         tokenizer_file=None,
         do_lower_case=True,
         remove_space=True,

src/transformers/models/barthez/tokenization_barthez_fast.py

@@ -109,7 +109,7 @@ class BarthezTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
+        vocab_file=None,
         tokenizer_file=None,
         bos_token="<s>",
         eos_token="</s>",

src/transformers/models/bert/tokenization_bert_fast.py

@@ -162,7 +162,7 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
+        vocab_file=None,
         tokenizer_file=None,
         do_lower_case=True,
         unk_token="[UNK]",

src/transformers/models/big_bird/tokenization_big_bird_fast.py

@@ -103,7 +103,7 @@ class BigBirdTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
+        vocab_file=None,
         tokenizer_file=None,
         unk_token="<unk>",
         bos_token="<s>",

src/transformers/models/blenderbot_small/tokenization_blenderbot_small_fast.py

@@ -63,8 +63,8 @@ class BlenderbotSmallTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
-        merges_file,
+        vocab_file=None,
+        merges_file=None,
         unk_token="<|endoftext|>",
         bos_token="<|endoftext|>",
         eos_token="<|endoftext|>",

src/transformers/models/camembert/tokenization_camembert_fast.py

@@ -105,7 +105,7 @@ class CamembertTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
+        vocab_file=None,
         tokenizer_file=None,
         bos_token="<s>",
         eos_token="</s>",

src/transformers/models/clip/tokenization_clip_fast.py

@@ -105,8 +105,8 @@ class CLIPTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
-        merges_file,
+        vocab_file=None,
+        merges_file=None,
         tokenizer_file=None,
         unk_token="<|endoftext|>",
         bos_token="<|startoftext|>",

src/transformers/models/deberta/tokenization_deberta_fast.py

@@ -95,8 +95,8 @@ class DebertaTokenizerFast(GPT2TokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
-        merges_file,
+        vocab_file=None,
+        merges_file=None,
         tokenizer_file=None,
         errors="replace",
         bos_token="[CLS]",

src/transformers/models/funnel/tokenization_funnel_fast.py

@@ -88,7 +88,7 @@ class FunnelTokenizerFast(BertTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
+        vocab_file=None,
         tokenizer_file=None,
         do_lower_case=True,
         unk_token="<unk>",

src/transformers/models/gpt2/tokenization_gpt2_fast.py

@@ -125,8 +125,8 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
-        merges_file,
+        vocab_file=None,
+        merges_file=None,
         tokenizer_file=None,
         unk_token="<|endoftext|>",
         bos_token="<|endoftext|>",

src/transformers/models/herbert/tokenization_herbert_fast.py

@@ -67,8 +67,8 @@ class HerbertTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
-        merges_file,
+        vocab_file=None,
+        merges_file=None,
         tokenizer_file=None,
         cls_token="<s>",
         unk_token="<unk>",

src/transformers/models/mbart/tokenization_mbart.py

@@ -121,7 +121,10 @@ class MBartTokenizer(XLMRobertaTokenizer):
         self._additional_special_tokens = list(self.lang_code_to_id.keys())
 
         if additional_special_tokens is not None:
-            self._additional_special_tokens.extend(additional_special_tokens)
+            # Only add those special tokens if they are not already there.
+            self._additional_special_tokens.extend(
+                [t for t in additional_special_tokens if t not in self._additional_special_tokens]
+            )
 
         self._src_lang = src_lang if src_lang is not None else "en_XX"
         self.cur_lang_code_id = self.lang_code_to_id[self._src_lang]

src/transformers/models/mbart/tokenization_mbart50_fast.py

@@ -110,7 +110,7 @@ class MBart50TokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
+        vocab_file=None,
         src_lang=None,
         tgt_lang=None,
         tokenizer_file=None,

src/transformers/models/mbart/tokenization_mbart_fast.py

@@ -113,10 +113,16 @@ class MBartTokenizerFast(XLMRobertaTokenizerFast):
     suffix_tokens: List[int] = []
 
     def __init__(
-        self, *args, tokenizer_file=None, src_lang=None, tgt_lang=None, additional_special_tokens=None, **kwargs
+        self,
+        vocab_file=None,
+        tokenizer_file=None,
+        src_lang=None,
+        tgt_lang=None,
+        additional_special_tokens=None,
+        **kwargs
     ):
         super().__init__(
-            *args,
+            vocab_file=vocab_file,
             tokenizer_file=tokenizer_file,
             src_lang=src_lang,
             tgt_lang=tgt_lang,
@@ -127,7 +133,10 @@ class MBartTokenizerFast(XLMRobertaTokenizerFast):
         _additional_special_tokens = FAIRSEQ_LANGUAGE_CODES.copy()
 
         if additional_special_tokens is not None:
-            _additional_special_tokens.extend(additional_special_tokens)
+            # Only add those special tokens if they are not already there.
+            _additional_special_tokens.extend(
+                [t for t in additional_special_tokens if t not in _additional_special_tokens]
+            )
 
         self.add_special_tokens({"additional_special_tokens": _additional_special_tokens})
 

src/transformers/models/mpnet/tokenization_mpnet_fast.py

@@ -106,7 +106,7 @@ class MPNetTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
+        vocab_file=None,
         tokenizer_file=None,
         do_lower_case=True,
         bos_token="<s>",

src/transformers/models/openai/tokenization_openai_fast.py

@@ -64,7 +64,7 @@ class OpenAIGPTTokenizerFast(PreTrainedTokenizerFast):
     model_input_names = ["input_ids", "attention_mask"]
     slow_tokenizer_class = OpenAIGPTTokenizer
 
-    def __init__(self, vocab_file, merges_file, tokenizer_file=None, unk_token="<unk>", **kwargs):
+    def __init__(self, vocab_file=None, merges_file=None, tokenizer_file=None, unk_token="<unk>", **kwargs):
         super().__init__(vocab_file, merges_file, tokenizer_file=tokenizer_file, unk_token=unk_token, **kwargs)
 
     @property

src/transformers/models/pegasus/tokenization_pegasus_fast.py

@@ -98,7 +98,7 @@ class PegasusTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
+        vocab_file=None,
         tokenizer_file=None,
         pad_token="<pad>",
         eos_token="</s>",

src/transformers/models/reformer/tokenization_reformer_fast.py

@@ -87,7 +87,7 @@ class ReformerTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
+        vocab_file=None,
         tokenizer_file=None,
         eos_token="</s>",
         unk_token="<unk>",

src/transformers/models/roberta/tokenization_roberta_fast.py

@@ -143,8 +143,8 @@ class RobertaTokenizerFast(GPT2TokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
-        merges_file,
+        vocab_file=None,
+        merges_file=None,
         tokenizer_file=None,
         errors="replace",
         bos_token="<s>",

src/transformers/models/roformer/tokenization_roformer_fast.py

@@ -73,7 +73,7 @@ class RoFormerTokenizerFast(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_file,
+        vocab_file=None,
         tokenizer_file=None,
         do_lower_case=True,
         unk_token="[UNK]",