Skip to content

GitLab

"...git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "cb47293eba06f88e320847248541a4713ed4dcdb"
Unverified Commit 7ec1dc88 authored by Yih-Dar's avatar Yih-Dar Committed by GitHub
Browse files

Add `RoCBertTokenizer` to `TOKENIZER_MAPPING_NAMES` (#20141) 



* Add RoCBertTokenizer to TOKENIZER_MAPPING_NAMES

* fix
Co-authored-by: default avatarydshieh <ydshieh@users.noreply.github.com>
parent 67b37891
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 0 deletions
src/transformers/models/auto/tokenization_auto.py
View file @ 7ec1dc88
...@@ -248,6 +248,7 @@ else: ...@@ -248,6 +248,7 @@ else:
), ),
("retribert", ("RetriBertTokenizer", "RetriBertTokenizerFast" if is_tokenizers_available() else None)), ("retribert", ("RetriBertTokenizer", "RetriBertTokenizerFast" if is_tokenizers_available() else None)),
("roberta", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)), ("roberta", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)),
("roc_bert", ("RoCBertTokenizer", None)),
("roformer", ("RoFormerTokenizer", "RoFormerTokenizerFast" if is_tokenizers_available() else None)), ("roformer", ("RoFormerTokenizer", "RoFormerTokenizerFast" if is_tokenizers_available() else None)),
("speech_to_text", ("Speech2TextTokenizer" if is_sentencepiece_available() else None, None)), ("speech_to_text", ("Speech2TextTokenizer" if is_sentencepiece_available() else None, None)),
("speech_to_text_2", ("Speech2Text2Tokenizer", None)), ("speech_to_text_2", ("Speech2Text2Tokenizer", None)),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment