"examples/git@developer.sourcefind.cn:OpenDAS/megatron-lm.git" did not exist on "290ad54d6c974aced88e34af7a8380edc05cc84c"
Unverified Commit b7cdd00f authored by Lysandre Debut's avatar Lysandre Debut Committed by GitHub
Browse files

Fix interaction of return_token_type_ids and add_special_tokens (#8854)

parent 04c446f7
......@@ -2814,7 +2814,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
len_ids = len(ids)
len_pair_ids = len(pair_ids) if pair else 0
if return_token_type_ids is not None and not add_special_tokens:
if return_token_type_ids and not add_special_tokens:
raise ValueError(
"Asking to return token_type_ids while setting add_special_tokens to False "
"results in an undefined behavior. Please set add_special_tokens to True or "
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment