Unverified Commit fcf10214 authored by Suraj Patil's avatar Suraj Patil Committed by GitHub
Browse files

enable loading Mbart50Tokenizer with AutoTokenizer (#10690)

* enable auto tokenizer for mbart50 tokenizers

* fix imports
parent bd8f6caf
...@@ -116,6 +116,7 @@ if is_sentencepiece_available(): ...@@ -116,6 +116,7 @@ if is_sentencepiece_available():
from ..m2m_100 import M2M100Tokenizer from ..m2m_100 import M2M100Tokenizer
from ..marian.tokenization_marian import MarianTokenizer from ..marian.tokenization_marian import MarianTokenizer
from ..mbart.tokenization_mbart import MBartTokenizer from ..mbart.tokenization_mbart import MBartTokenizer
from ..mbart.tokenization_mbart50 import MBart50Tokenizer
from ..mt5 import MT5Tokenizer from ..mt5 import MT5Tokenizer
from ..pegasus.tokenization_pegasus import PegasusTokenizer from ..pegasus.tokenization_pegasus import PegasusTokenizer
from ..reformer.tokenization_reformer import ReformerTokenizer from ..reformer.tokenization_reformer import ReformerTokenizer
...@@ -132,6 +133,7 @@ else: ...@@ -132,6 +133,7 @@ else:
DebertaV2Tokenizer = None DebertaV2Tokenizer = None
MarianTokenizer = None MarianTokenizer = None
MBartTokenizer = None MBartTokenizer = None
MBart50Tokenizer = None
MT5Tokenizer = None MT5Tokenizer = None
PegasusTokenizer = None PegasusTokenizer = None
ReformerTokenizer = None ReformerTokenizer = None
...@@ -159,6 +161,7 @@ if is_tokenizers_available(): ...@@ -159,6 +161,7 @@ if is_tokenizers_available():
from ..led.tokenization_led_fast import LEDTokenizerFast from ..led.tokenization_led_fast import LEDTokenizerFast
from ..longformer.tokenization_longformer_fast import LongformerTokenizerFast from ..longformer.tokenization_longformer_fast import LongformerTokenizerFast
from ..lxmert.tokenization_lxmert_fast import LxmertTokenizerFast from ..lxmert.tokenization_lxmert_fast import LxmertTokenizerFast
from ..mbart.tokenization_mbart50_fast import MBart50TokenizerFast
from ..mbart.tokenization_mbart_fast import MBartTokenizerFast from ..mbart.tokenization_mbart_fast import MBartTokenizerFast
from ..mobilebert.tokenization_mobilebert_fast import MobileBertTokenizerFast from ..mobilebert.tokenization_mobilebert_fast import MobileBertTokenizerFast
from ..mpnet.tokenization_mpnet_fast import MPNetTokenizerFast from ..mpnet.tokenization_mpnet_fast import MPNetTokenizerFast
...@@ -190,6 +193,7 @@ else: ...@@ -190,6 +193,7 @@ else:
LongformerTokenizerFast = None LongformerTokenizerFast = None
LxmertTokenizerFast = None LxmertTokenizerFast = None
MBartTokenizerFast = None MBartTokenizerFast = None
MBart50TokenizerFast = None
MobileBertTokenizerFast = None MobileBertTokenizerFast = None
MPNetTokenizerFast = None MPNetTokenizerFast = None
MT5TokenizerFast = None MT5TokenizerFast = None
...@@ -268,6 +272,8 @@ NO_CONFIG_TOKENIZER = [ ...@@ -268,6 +272,8 @@ NO_CONFIG_TOKENIZER = [
PhobertTokenizer, PhobertTokenizer,
BarthezTokenizer, BarthezTokenizer,
BarthezTokenizerFast, BarthezTokenizerFast,
MBart50Tokenizer,
MBart50TokenizerFast,
] ]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment