Add PRETRAINED_INIT_CONFIGURATION to DistilBERT tokenizer

801f2ac8 · Lysandre · Lysandre Debut · bfec203d · 801f2ac8
Commit 801f2ac8 authored Jan 09, 2020 by Lysandre Committed by Lysandre Debut Jan 10, 2020
Hide whitespace changes
Inline Side-by-side

Showing with 9 additions and 0 deletions

src/transformers/tokenization_distilbert.py src/transformers/tokenization_distilbert.py +9 -0

No files found.
--- a/src/transformers/tokenization_distilbert.py
+++ b/src/transformers/tokenization_distilbert.py
@@ -41,6 +41,14 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
 }


+PRETRAINED_INIT_CONFIGURATION = {
+    "distilbert-base-uncased": {"do_lower_case": True},
+    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
+    "distilbert-base-german-cased": {"do_lower_case": False},
+    "distilbert-base-multilingual-cased": {"do_lower_case": False},
+}
+
+
 class DistilBertTokenizer(BertTokenizer):
    r"""
    Constructs a DistilBertTokenizer.
@@ -59,3 +67,4 @@ class DistilBertTokenizer(BertTokenizer):
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION