Skip to content

GitLab

Commit d6160224 authored by thomwolf's avatar thomwolf
Browse files

fix openai special tokens loading

parent df5d9c35
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 0 deletions
pytorch_pretrained_bert/tokenization_openai.py
View file @ d6160224
...@@ -87,6 +87,7 @@ class OpenAIGPTTokenizer(object): ...@@ -87,6 +87,7 @@ class OpenAIGPTTokenizer(object):
if pretrained_model_name_or_path in PRETRAINED_VOCAB_ARCHIVE_MAP: if pretrained_model_name_or_path in PRETRAINED_VOCAB_ARCHIVE_MAP:
vocab_file = PRETRAINED_VOCAB_ARCHIVE_MAP[pretrained_model_name_or_path] vocab_file = PRETRAINED_VOCAB_ARCHIVE_MAP[pretrained_model_name_or_path]
merges_file = PRETRAINED_MERGES_ARCHIVE_MAP[pretrained_model_name_or_path] merges_file = PRETRAINED_MERGES_ARCHIVE_MAP[pretrained_model_name_or_path]
special_tokens_file = None
else: else:
vocab_file = os.path.join(pretrained_model_name_or_path, VOCAB_NAME) vocab_file = os.path.join(pretrained_model_name_or_path, VOCAB_NAME)
merges_file = os.path.join(pretrained_model_name_or_path, MERGES_NAME) merges_file = os.path.join(pretrained_model_name_or_path, MERGES_NAME)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment