Unverified Commit b7cdd00f authored by Lysandre Debut's avatar Lysandre Debut Committed by GitHub
Browse files

Fix interaction of return_token_type_ids and add_special_tokens (#8854)

parent 04c446f7
...@@ -2814,7 +2814,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin): ...@@ -2814,7 +2814,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
len_ids = len(ids) len_ids = len(ids)
len_pair_ids = len(pair_ids) if pair else 0 len_pair_ids = len(pair_ids) if pair else 0
if return_token_type_ids is not None and not add_special_tokens: if return_token_type_ids and not add_special_tokens:
raise ValueError( raise ValueError(
"Asking to return token_type_ids while setting add_special_tokens to False " "Asking to return token_type_ids while setting add_special_tokens to False "
"results in an undefined behavior. Please set add_special_tokens to True or " "results in an undefined behavior. Please set add_special_tokens to True or "
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment