"vscode:/vscode.git/clone" did not exist on "ed3bd0d4282e27f3698e35b38ffddb4dbb1fda76"
Unverified Commit f20aec1d authored by Sam Shleifer's avatar Sam Shleifer Committed by GitHub
Browse files

fsmt slow test uses lists (#8031)

parent 101186bc
......@@ -144,11 +144,11 @@ class FSMTTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
# for src_text, _ in targets: print(f"""[\n"{src_text}",\n {model.encode(src_text).tolist()}\n],""")
for src_text, tgt_input_ids in targets:
input_ids = tokenizer_enc.encode(src_text, return_tensors="pt")[0].tolist()
self.assertListEqual(input_ids, tgt_input_ids)
encoded_ids = tokenizer_enc.encode(src_text, return_tensors=None)
self.assertListEqual(encoded_ids, tgt_input_ids)
# and decode backward, using the reversed languages model
decoded_text = tokenizer_dec.decode(input_ids, skip_special_tokens=True)
decoded_text = tokenizer_dec.decode(encoded_ids, skip_special_tokens=True)
self.assertEqual(decoded_text, src_text)
@unittest.skip("FSMTConfig.__init__ requires non-optional args")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment