"vscode:/vscode.git/clone" did not exist on "3213d3bfaec079e03007e3898648b4eafd00d378"
Commit f39217a5 authored by Julien Chaumond's avatar Julien Chaumond
Browse files

[tests] Light cleanup of tempfile in tests/

parent f54dc3f4
...@@ -107,15 +107,14 @@ class TokenizerTesterMixin: ...@@ -107,15 +107,14 @@ class TokenizerTesterMixin:
before_tokens = tokenizer.encode("He is very happy, UNwant\u00E9d,running", add_special_tokens=False) before_tokens = tokenizer.encode("He is very happy, UNwant\u00E9d,running", add_special_tokens=False)
with tempfile.TemporaryDirectory() as tmpdirname: tokenizer.save_pretrained(self.tmpdirname)
tokenizer.save_pretrained(tmpdirname) tokenizer = self.tokenizer_class.from_pretrained(self.tmpdirname)
tokenizer = self.tokenizer_class.from_pretrained(tmpdirname)
after_tokens = tokenizer.encode("He is very happy, UNwant\u00E9d,running", add_special_tokens=False) after_tokens = tokenizer.encode("He is very happy, UNwant\u00E9d,running", add_special_tokens=False)
self.assertListEqual(before_tokens, after_tokens) self.assertListEqual(before_tokens, after_tokens)
self.assertEqual(tokenizer.max_len, 42) self.assertEqual(tokenizer.max_len, 42)
tokenizer = self.tokenizer_class.from_pretrained(tmpdirname, max_len=43) tokenizer = self.tokenizer_class.from_pretrained(self.tmpdirname, max_len=43)
self.assertEqual(tokenizer.max_len, 43) self.assertEqual(tokenizer.max_len, 43)
def test_pickle_tokenizer(self): def test_pickle_tokenizer(self):
...@@ -125,9 +124,7 @@ class TokenizerTesterMixin: ...@@ -125,9 +124,7 @@ class TokenizerTesterMixin:
text = "Munich and Berlin are nice cities" text = "Munich and Berlin are nice cities"
subwords = tokenizer.tokenize(text) subwords = tokenizer.tokenize(text)
with tempfile.TemporaryDirectory() as tmpdirname: filename = os.path.join(self.tmpdirname, "tokenizer.bin")
filename = os.path.join(tmpdirname, "tokenizer.bin")
with open(filename, "wb") as handle: with open(filename, "wb") as handle:
pickle.dump(tokenizer, handle) pickle.dump(tokenizer, handle)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment