Commit f39217a5 authored by Julien Chaumond's avatar Julien Chaumond
Browse files

[tests] Light cleanup of tempfile in tests/

parent f54dc3f4
......@@ -107,15 +107,14 @@ class TokenizerTesterMixin:
before_tokens = tokenizer.encode("He is very happy, UNwant\u00E9d,running", add_special_tokens=False)
with tempfile.TemporaryDirectory() as tmpdirname:
tokenizer.save_pretrained(tmpdirname)
tokenizer = self.tokenizer_class.from_pretrained(tmpdirname)
tokenizer.save_pretrained(self.tmpdirname)
tokenizer = self.tokenizer_class.from_pretrained(self.tmpdirname)
after_tokens = tokenizer.encode("He is very happy, UNwant\u00E9d,running", add_special_tokens=False)
self.assertListEqual(before_tokens, after_tokens)
self.assertEqual(tokenizer.max_len, 42)
tokenizer = self.tokenizer_class.from_pretrained(tmpdirname, max_len=43)
tokenizer = self.tokenizer_class.from_pretrained(self.tmpdirname, max_len=43)
self.assertEqual(tokenizer.max_len, 43)
def test_pickle_tokenizer(self):
......@@ -125,9 +124,7 @@ class TokenizerTesterMixin:
text = "Munich and Berlin are nice cities"
subwords = tokenizer.tokenize(text)
with tempfile.TemporaryDirectory() as tmpdirname:
filename = os.path.join(tmpdirname, "tokenizer.bin")
filename = os.path.join(self.tmpdirname, "tokenizer.bin")
with open(filename, "wb") as handle:
pickle.dump(tokenizer, handle)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment