Skip to content

GitLab

Unverified Commit c244562c authored by Thomas Wolf's avatar Thomas Wolf Committed by GitHub
Browse files

Merge pull request #893 from joelgrus/patch-2 

make save_pretrained do the right thing with added tokens
parents 067923d3 ae152cec
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
pytorch_transformers/tokenization_utils.py
View file @ c244562c
...@@ -266,7 +266,7 @@ class PreTrainedTokenizer(object): ...@@ -266,7 +266,7 @@ class PreTrainedTokenizer(object):
with open(added_tokens_file, 'w', encoding='utf-8') as f: with open(added_tokens_file, 'w', encoding='utf-8') as f:
if self.added_tokens_encoder: if self.added_tokens_encoder:
out_str = json.dumps(self.added_tokens_decoder, ensure_ascii=False) out_str = json.dumps(self.added_tokens_encoder, ensure_ascii=False)
else: else:
out_str = u"{}" out_str = u"{}"
f.write(out_str) f.write(out_str)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment