Skip to content

GitLab

Commit 31d3373b authored by LysandreJik's avatar LysandreJik
Browse files

Appends space before special token

parent fede4ef4
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
pytorch_transformers/tokenization_utils.py
View file @ 31d3373b
...@@ -760,7 +760,7 @@ class PreTrainedTokenizer(object): ...@@ -760,7 +760,7 @@ class PreTrainedTokenizer(object):
if current_sub_text: if current_sub_text:
sub_texts.append(self.convert_tokens_to_string(current_sub_text)) sub_texts.append(self.convert_tokens_to_string(current_sub_text))
current_sub_text = [] current_sub_text = []
sub_texts.append(token) sub_texts.append(" " + token)
else: else:
current_sub_text.append(token) current_sub_text.append(token)
if current_sub_text: if current_sub_text:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment