Commit c4d4f3ec authored by LysandreJik's avatar LysandreJik
Browse files

Updated DistilBERT test to reflect the sequence encoding

parent d572d702
......@@ -39,8 +39,8 @@ class DistilBertTokenizationTest(BertTokenizationTest):
encoded_sentence = tokenizer.add_special_tokens_single_sentence(text)
encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2)
assert encoded_sentence == [101] + text + [102]
assert encoded_pair == [101] + text + [102] + text_2 + [102]
assert encoded_sentence == text
assert encoded_pair == text + [102] + text_2
if __name__ == '__main__':
unittest.main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment