fix openai special tokens loading

d6160224 · thomwolf · df5d9c35 · d6160224
Commit d6160224 authored Apr 15, 2019 by thomwolf
Show whitespace changes
Inline Side-by-side

Showing with 1 addition and 0 deletions

pytorch_pretrained_bert/tokenization_openai.py pytorch_pretrained_bert/tokenization_openai.py +1 -0

No files found.
--- a/pytorch_pretrained_bert/tokenization_openai.py
+++ b/pytorch_pretrained_bert/tokenization_openai.py
@@ -87,6 +87,7 @@ class OpenAIGPTTokenizer(object):
        if pretrained_model_name_or_path in PRETRAINED_VOCAB_ARCHIVE_MAP:
            vocab_file = PRETRAINED_VOCAB_ARCHIVE_MAP[pretrained_model_name_or_path]
            merges_file = PRETRAINED_MERGES_ARCHIVE_MAP[pretrained_model_name_or_path]
+            special_tokens_file = None
        else:
            vocab_file = os.path.join(pretrained_model_name_or_path, VOCAB_NAME)
            merges_file = os.path.join(pretrained_model_name_or_path, MERGES_NAME)