Skip to content

GitLab

Unverified Commit f51161e2 authored by Suraj Patil's avatar Suraj Patil Committed by GitHub
Browse files

add BartTokenizerFast in AutoTokenizer (#6464) 


Co-authored-by: default avatarLysandre Debut <lysandre@huggingface.co>
parent a442f87a
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
src/transformers/tokenization_auto.py
View file @ f51161e2
...@@ -46,7 +46,7 @@ from .configuration_auto import ( ...@@ -46,7 +46,7 @@ from .configuration_auto import (
) )
from .configuration_utils import PretrainedConfig from .configuration_utils import PretrainedConfig
from .tokenization_albert import AlbertTokenizer from .tokenization_albert import AlbertTokenizer
from .tokenization_bart import BartTokenizer, MBartTokenizer from .tokenization_bart import BartTokenizer, BartTokenizerFast, MBartTokenizer
from .tokenization_bert import BertTokenizer, BertTokenizerFast from .tokenization_bert import BertTokenizer, BertTokenizerFast
from .tokenization_bert_japanese import BertJapaneseTokenizer from .tokenization_bert_japanese import BertJapaneseTokenizer
from .tokenization_camembert import CamembertTokenizer from .tokenization_camembert import CamembertTokenizer
...@@ -85,7 +85,7 @@ TOKENIZER_MAPPING = OrderedDict( ...@@ -85,7 +85,7 @@ TOKENIZER_MAPPING = OrderedDict(
(MBartConfig, (MBartTokenizer, None)), (MBartConfig, (MBartTokenizer, None)),
(XLMRobertaConfig, (XLMRobertaTokenizer, None)), (XLMRobertaConfig, (XLMRobertaTokenizer, None)),
(MarianConfig, (MarianTokenizer, None)), (MarianConfig, (MarianTokenizer, None)),
(BartConfig, (BartTokenizer, None)), (BartConfig, (BartTokenizer, BartTokenizerFast)),
(LongformerConfig, (LongformerTokenizer, LongformerTokenizerFast)), (LongformerConfig, (LongformerTokenizer, LongformerTokenizerFast)),
(RobertaConfig, (RobertaTokenizer, RobertaTokenizerFast)), (RobertaConfig, (RobertaTokenizer, RobertaTokenizerFast)),
(ReformerConfig, (ReformerTokenizer, None)), (ReformerConfig, (ReformerTokenizer, None)),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment