tokenization add XLM-RoBERTa base model

128cfdee · Stefan Schweter · e778dd85 · 128cfdee
Commit 128cfdee authored Dec 18, 2019 by Stefan Schweter
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 0 deletions

transformers/tokenization_xlm_roberta.py transformers/tokenization_xlm_roberta.py +2 -0

No files found.
--- a/transformers/tokenization_xlm_roberta.py
+++ b/transformers/tokenization_xlm_roberta.py
@@ -30,11 +30,13 @@ VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'}
 PRETRAINED_VOCAB_FILES_MAP = {
    'vocab_file':
    {
+    'xlm-roberta-base': "https://schweter.eu/cloud/transformers/xlm-roberta-base-sentencepiece.bpe.model",
    'xlm-roberta-large': "https://schweter.eu/cloud/transformers/xlm-roberta-large-sentencepiece.bpe.model",
    }
 }
 PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    'xlm-roberta-base': None,
    'xlm-roberta-large': None,
 }