Unverified Commit ae152cec authored by Joel Grus's avatar Joel Grus Committed by GitHub
Browse files

make save_pretrained work with added tokens

right now it's dumping the *decoder* when it should be dumping the *encoder*. this fixes that.
parent 067923d3
......@@ -266,7 +266,7 @@ class PreTrainedTokenizer(object):
with open(added_tokens_file, 'w', encoding='utf-8') as f:
if self.added_tokens_encoder:
out_str = json.dumps(self.added_tokens_decoder, ensure_ascii=False)
out_str = json.dumps(self.added_tokens_encoder, ensure_ascii=False)
else:
out_str = u"{}"
f.write(out_str)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment