Skip to content

GitLab

Commit ce5ef4b3 authored by thomwolf's avatar thomwolf
Browse files

python2 doesn't spark joy

parent 5dd7b677
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
pytorch_transformers/tests/tokenization_gpt2_test.py
View file @ ce5ef4b3
...@@ -42,7 +42,7 @@ class GPT2TokenizationTest(CommonTestCases.CommonTokenizerTester): ...@@ -42,7 +42,7 @@ class GPT2TokenizationTest(CommonTestCases.CommonTokenizerTester):
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file']) self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file'])
self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['merges_file']) self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['merges_file'])
with open(self.vocab_file, "w", encoding="utf-8") as fp: with open(self.vocab_file, "w", encoding="utf-8") as fp:
fp.write(json.dumps(vocab_tokens)) fp.write(json.dumps(vocab_tokens) + "\n")
with open(self.merges_file, "w", encoding="utf-8") as fp: with open(self.merges_file, "w", encoding="utf-8") as fp:
fp.write("\n".join(merges)) fp.write("\n".join(merges))
......
pytorch_transformers/tests/tokenization_roberta_test.py
View file @ ce5ef4b3
...@@ -41,7 +41,7 @@ class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester): ...@@ -41,7 +41,7 @@ class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester):
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file']) self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file'])
self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['merges_file']) self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['merges_file'])
with open(self.vocab_file, "w", encoding="utf-8") as fp: with open(self.vocab_file, "w", encoding="utf-8") as fp:
fp.write(json.dumps(vocab_tokens)) fp.write(json.dumps(vocab_tokens) + "\n")
with open(self.merges_file, "w", encoding="utf-8") as fp: with open(self.merges_file, "w", encoding="utf-8") as fp:
fp.write("\n".join(merges)) fp.write("\n".join(merges))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment