Skip to content

GitLab

Commit 801f2ac8 authored by Lysandre's avatar Lysandre Committed by Lysandre Debut
Browse files

Add PRETRAINED_INIT_CONFIGURATION to DistilBERT tokenizer

parent bfec203d
Hide whitespace changes
Inline Side-by-side
Showing with 9 additions and 0 deletions
src/transformers/tokenization_distilbert.py
View file @ 801f2ac8
...@@ -41,6 +41,14 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { ...@@ -41,6 +41,14 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
} }
PRETRAINED_INIT_CONFIGURATION = {
"distilbert-base-uncased": {"do_lower_case": True},
"distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
"distilbert-base-german-cased": {"do_lower_case": False},
"distilbert-base-multilingual-cased": {"do_lower_case": False},
}
class DistilBertTokenizer(BertTokenizer): class DistilBertTokenizer(BertTokenizer):
r""" r"""
Constructs a DistilBertTokenizer. Constructs a DistilBertTokenizer.
...@@ -59,3 +67,4 @@ class DistilBertTokenizer(BertTokenizer): ...@@ -59,3 +67,4 @@ class DistilBertTokenizer(BertTokenizer):
vocab_files_names = VOCAB_FILES_NAMES vocab_files_names = VOCAB_FILES_NAMES
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment