Unverified Commit 7ec1dc88 authored by Yih-Dar's avatar Yih-Dar Committed by GitHub
Browse files

Add `RoCBertTokenizer` to `TOKENIZER_MAPPING_NAMES` (#20141)



* Add RoCBertTokenizer to TOKENIZER_MAPPING_NAMES

* fix
Co-authored-by: default avatarydshieh <ydshieh@users.noreply.github.com>
parent 67b37891
...@@ -248,6 +248,7 @@ else: ...@@ -248,6 +248,7 @@ else:
), ),
("retribert", ("RetriBertTokenizer", "RetriBertTokenizerFast" if is_tokenizers_available() else None)), ("retribert", ("RetriBertTokenizer", "RetriBertTokenizerFast" if is_tokenizers_available() else None)),
("roberta", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)), ("roberta", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)),
("roc_bert", ("RoCBertTokenizer", None)),
("roformer", ("RoFormerTokenizer", "RoFormerTokenizerFast" if is_tokenizers_available() else None)), ("roformer", ("RoFormerTokenizer", "RoFormerTokenizerFast" if is_tokenizers_available() else None)),
("speech_to_text", ("Speech2TextTokenizer" if is_sentencepiece_available() else None, None)), ("speech_to_text", ("Speech2TextTokenizer" if is_sentencepiece_available() else None, None)),
("speech_to_text_2", ("Speech2Text2Tokenizer", None)), ("speech_to_text_2", ("Speech2Text2Tokenizer", None)),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment