add BartTokenizerFast in AutoTokenizer (#6464)

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>

add BartTokenizerFast in AutoTokenizer (#6464)
Co-authored-by: Lysandre Debut <lysandre@huggingface.co>
f51161e2 · Suraj Patil · GitHub · a442f87a · f51161e2
Unverified Commit f51161e2 authored Aug 13, 2020 by Suraj Patil Committed by GitHub Aug 13, 2020
Show whitespace changes
Inline Side-by-side

Showing with 2 additions and 2 deletions

src/transformers/tokenization_auto.py src/transformers/tokenization_auto.py +2 -2

No files found.
--- a/src/transformers/tokenization_auto.py
+++ b/src/transformers/tokenization_auto.py
@@ -46,7 +46,7 @@ from .configuration_auto import (
 )
 from .configuration_utils import PretrainedConfig
 from .tokenization_albert import AlbertTokenizer
-from .tokenization_bart import BartTokenizer, MBartTokenizer
+from .tokenization_bart import BartTokenizer, BartTokenizerFast, MBartTokenizer
 from .tokenization_bert import BertTokenizer, BertTokenizerFast
 from .tokenization_bert_japanese import BertJapaneseTokenizer
 from .tokenization_camembert import CamembertTokenizer
@@ -85,7 +85,7 @@ TOKENIZER_MAPPING = OrderedDict(
        (MBartConfig, (MBartTokenizer, None)),
        (XLMRobertaConfig, (XLMRobertaTokenizer, None)),
        (MarianConfig, (MarianTokenizer, None)),
-        (BartConfig, (BartTokenizer, None)),
+        (BartConfig, (BartTokenizer, BartTokenizerFast)),
        (LongformerConfig, (LongformerTokenizer, LongformerTokenizerFast)),
        (RobertaConfig, (RobertaTokenizer, RobertaTokenizerFast)),
        (ReformerConfig, (ReformerTokenizer, None)),