Skip to content

GitLab

Commit 7f998b1b authored by Guillaume B's avatar Guillaume B
Browse files

special_tokens_mask value was unused and calculated twice

parent fb0d2f1d
Show whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
transformers/tokenization_utils.py
View file @ 7f998b1b
...@@ -910,7 +910,7 @@ class PreTrainedTokenizer(object): ...@@ -910,7 +910,7 @@ class PreTrainedTokenizer(object):
token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else [])
special_tokens_mask = [0] * (len(ids) + (len(pair_ids) if pair else 0)) special_tokens_mask = [0] * (len(ids) + (len(pair_ids) if pair else 0))
if return_special_tokens_mask: if return_special_tokens_mask:
encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) encoded_inputs["special_tokens_mask"] = special_tokens_mask
# Prepare inputs as tensors if asked # Prepare inputs as tensors if asked
if return_tensors == 'tf' and is_tf_available(): if return_tensors == 'tf' and is_tf_available():
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment