Unverified Commit 39a72125 authored by Arthur's avatar Arthur Committed by GitHub
Browse files

fix both failing RoCBert tests (#20469)

parent 30163921
......@@ -710,4 +710,4 @@ class RoCBertModelIntegrationTest(unittest.TestCase):
# convert to tokens is: ['[CLS]', '巴', '*', '黎', '是', '法', '国', '的', '首', '都', '[SEP]']
expected_output = torch.tensor([[101, 2349, 115, 7944, 3221, 3791, 1744, 4638, 7674, 6963, 102]])
self.assertEqual(output_ids, expected_output)
assert torch.allclose(output_ids, expected_output)
......@@ -299,8 +299,8 @@ class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
encoded_sentence = tokenizer.build_inputs_with_special_tokens(text)
encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2)
assert encoded_sentence == [101] + text + [102]
assert encoded_pair == [101] + text + [102] + text_2 + [102]
assert encoded_sentence == [1] + text + [2]
assert encoded_pair == [1] + text + [2] + text_2 + [2]
def test_prepare_for_model(self):
tokenizers = self.get_tokenizers(do_lower_case=False)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment