Skip to content

GitLab

"...git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "173528e3685bc4321630b7f979d01896c57a5c15"
Unverified Commit 90df44f0 authored by Thomas Wolf's avatar Thomas Wolf Committed by GitHub
Browse files

Merge pull request #2063 from guillaume-be/special_tokens_mask_value_not_used 

special_tokens_mask value was unused and calculated twice
parents 707f9e92 7f998b1b
Show whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
transformers/tokenization_utils.py
View file @ 90df44f0
...@@ -973,7 +973,7 @@ class PreTrainedTokenizer(object): ...@@ -973,7 +973,7 @@ class PreTrainedTokenizer(object):
token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else [])
if return_special_tokens_mask: if return_special_tokens_mask:
encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) encoded_inputs["special_tokens_mask"] = special_tokens_mask
encoded_inputs["input_ids"] = sequence encoded_inputs["input_ids"] = sequence
if return_token_type_ids: if return_token_type_ids:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment