Skip to content

GitLab

Unverified Commit f51161e2 authored by Suraj Patil's avatar Suraj Patil Committed by GitHub
Browse files

add BartTokenizerFast in AutoTokenizer (#6464) 


Co-authored-by: default avatarLysandre Debut <lysandre@huggingface.co>
parent a442f87a
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
src/transformers/tokenization_auto.py
View file @ f51161e2
......@@ -46,7 +46,7 @@ from .configuration_auto import (
)
from .configuration_utils import PretrainedConfig
from .tokenization_albert import AlbertTokenizer
from .tokenization_bart import BartTokenizer, MBartTokenizer
from .tokenization_bart import BartTokenizer, BartTokenizerFast, MBartTokenizer
from .tokenization_bert import BertTokenizer, BertTokenizerFast
from .tokenization_bert_japanese import BertJapaneseTokenizer
from .tokenization_camembert import CamembertTokenizer
......@@ -85,7 +85,7 @@ TOKENIZER_MAPPING = OrderedDict(
(MBartConfig, (MBartTokenizer, None)),
(XLMRobertaConfig, (XLMRobertaTokenizer, None)),
(MarianConfig, (MarianTokenizer, None)),
(BartConfig, (BartTokenizer, None)),
(BartConfig, (BartTokenizer, BartTokenizerFast)),
(LongformerConfig, (LongformerTokenizer, LongformerTokenizerFast)),
(RobertaConfig, (RobertaTokenizer, RobertaTokenizerFast)),
(ReformerConfig, (ReformerTokenizer, None)),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment