"vscode:/vscode.git/clone" did not exist on "9fbb4cdc8074b4f74cae20ad68f4ba4f93fe0005"
Unverified Commit fcf10214 authored by Suraj Patil's avatar Suraj Patil Committed by GitHub
Browse files

enable loading Mbart50Tokenizer with AutoTokenizer (#10690)

* enable auto tokenizer for mbart50 tokenizers

* fix imports
parent bd8f6caf
......@@ -116,6 +116,7 @@ if is_sentencepiece_available():
from ..m2m_100 import M2M100Tokenizer
from ..marian.tokenization_marian import MarianTokenizer
from ..mbart.tokenization_mbart import MBartTokenizer
from ..mbart.tokenization_mbart50 import MBart50Tokenizer
from ..mt5 import MT5Tokenizer
from ..pegasus.tokenization_pegasus import PegasusTokenizer
from ..reformer.tokenization_reformer import ReformerTokenizer
......@@ -132,6 +133,7 @@ else:
DebertaV2Tokenizer = None
MarianTokenizer = None
MBartTokenizer = None
MBart50Tokenizer = None
MT5Tokenizer = None
PegasusTokenizer = None
ReformerTokenizer = None
......@@ -159,6 +161,7 @@ if is_tokenizers_available():
from ..led.tokenization_led_fast import LEDTokenizerFast
from ..longformer.tokenization_longformer_fast import LongformerTokenizerFast
from ..lxmert.tokenization_lxmert_fast import LxmertTokenizerFast
from ..mbart.tokenization_mbart50_fast import MBart50TokenizerFast
from ..mbart.tokenization_mbart_fast import MBartTokenizerFast
from ..mobilebert.tokenization_mobilebert_fast import MobileBertTokenizerFast
from ..mpnet.tokenization_mpnet_fast import MPNetTokenizerFast
......@@ -190,6 +193,7 @@ else:
LongformerTokenizerFast = None
LxmertTokenizerFast = None
MBartTokenizerFast = None
MBart50TokenizerFast = None
MobileBertTokenizerFast = None
MPNetTokenizerFast = None
MT5TokenizerFast = None
......@@ -268,6 +272,8 @@ NO_CONFIG_TOKENIZER = [
PhobertTokenizer,
BarthezTokenizer,
BarthezTokenizerFast,
MBart50Tokenizer,
MBart50TokenizerFast,
]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment