Skip to content

GitLab

Unverified Commit e739a361 authored by Arthur's avatar Arthur Committed by GitHub
Browse files

single word should be set to False (#27738)

parent 2b5d5ead
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
src/transformers/models/t5/tokenization_t5.py
View file @ e739a361
...@@ -183,7 +183,7 @@ class T5Tokenizer(PreTrainedTokenizer): ...@@ -183,7 +183,7 @@ class T5Tokenizer(PreTrainedTokenizer):
self._added_tokens_decoder = {} self._added_tokens_decoder = {}
for i in range(len(extra_tokens)): for i in range(len(extra_tokens)):
self._added_tokens_decoder[len(self.sp_model) - 1 + extra_ids - i] = AddedToken( self._added_tokens_decoder[len(self.sp_model) - 1 + extra_ids - i] = AddedToken(
f"<extra_id_{i}>", single_word=True, lstrip=True, rstrip=True, special=True f"<extra_id_{i}>", single_word=False, lstrip=True, rstrip=True, special=True, normalized=False
) )
if legacy is None: if legacy is None:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment