Unverified Commit 2886f7f0 authored by Yih-Dar's avatar Yih-Dar Committed by GitHub
Browse files

Fix tokenizer for XLMRobertaXL (#19004)


Co-authored-by: default avatarydshieh <ydshieh@users.noreply.github.com>
parent 2848c9ce
......@@ -271,7 +271,13 @@ else:
"XLMRobertaTokenizerFast" if is_tokenizers_available() else None,
),
),
("xlm-roberta-xl", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)),
(
"xlm-roberta-xl",
(
"XLMRobertaTokenizer" if is_sentencepiece_available() else None,
"XLMRobertaTokenizerFast" if is_tokenizers_available() else None,
),
),
(
"xlnet",
(
......
......@@ -54,7 +54,7 @@ logger = logging.get_logger(__name__)
_CHECKPOINT_FOR_DOC = "xlm-roberta-xlarge"
_CONFIG_FOR_DOC = "XLMRobertaXLConfig"
_TOKENIZER_FOR_DOC = "RobertaTokenizer"
_TOKENIZER_FOR_DOC = "XLMRobertaTokenizer"
XLM_ROBERTA_XL_PRETRAINED_MODEL_ARCHIVE_LIST = [
"facebook/xlm-roberta-xl",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment