Appends space before special token

31d3373b · LysandreJik · fede4ef4 · 31d3373b
Commit 31d3373b authored Sep 01, 2019 by LysandreJik
Show whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

pytorch_transformers/tokenization_utils.py pytorch_transformers/tokenization_utils.py +1 -1

No files found.
--- a/pytorch_transformers/tokenization_utils.py
+++ b/pytorch_transformers/tokenization_utils.py
@@ -760,7 +760,7 @@ class PreTrainedTokenizer(object):
                if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(current_sub_text))
                    current_sub_text = []
-                sub_texts.append(token)
+                sub_texts.append(" " + token)
            else:
                current_sub_text.append(token)
        if current_sub_text: