add LongformerTokenizerFast in AutoTokenizer (#6463)

a442f87a · Suraj Patil · GitHub · f7cbc13d · a442f87a
Unverified Commit a442f87a authored Aug 13, 2020 by Suraj Patil Committed by GitHub Aug 13, 2020
Show whitespace changes
Inline Side-by-side

Showing with 2 additions and 2 deletions

src/transformers/tokenization_auto.py src/transformers/tokenization_auto.py +2 -2

No files found.
--- a/src/transformers/tokenization_auto.py
+++ b/src/transformers/tokenization_auto.py
@@ -55,7 +55,7 @@ from .tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFas
 from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast
 from .tokenization_flaubert import FlaubertTokenizer
 from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
-from .tokenization_longformer import LongformerTokenizer
+from .tokenization_longformer import LongformerTokenizer, LongformerTokenizerFast
 from .tokenization_marian import MarianTokenizer
 from .tokenization_mobilebert import MobileBertTokenizer, MobileBertTokenizerFast
 from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
@@ -86,7 +86,7 @@ TOKENIZER_MAPPING = OrderedDict(
        (XLMRobertaConfig, (XLMRobertaTokenizer, None)),
        (MarianConfig, (MarianTokenizer, None)),
        (BartConfig, (BartTokenizer, None)),
-        (LongformerConfig, (LongformerTokenizer, None)),
+        (LongformerConfig, (LongformerTokenizer, LongformerTokenizerFast)),
        (RobertaConfig, (RobertaTokenizer, RobertaTokenizerFast)),
        (ReformerConfig, (ReformerTokenizer, None)),
        (ElectraConfig, (ElectraTokenizer, ElectraTokenizerFast)),