Skip to content

GitLab

Commit fe92755b authored by LysandreJik's avatar LysandreJik
Browse files

Fix special tokens mask in encode

parent fbf5455a
Show whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
transformers/tokenization_utils.py
View file @ fe92755b
...@@ -973,7 +973,7 @@ class PreTrainedTokenizer(object): ...@@ -973,7 +973,7 @@ class PreTrainedTokenizer(object):
token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else [])
if return_special_tokens_mask: if return_special_tokens_mask:
encoded_inputs["special_tokens_mask"] = special_tokens_mask encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids)
encoded_inputs["input_ids"] = sequence encoded_inputs["input_ids"] = sequence
if return_token_type_ids: if return_token_type_ids:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment