Skip to content

GitLab

Commit 850da1cc authored by thomwolf's avatar thomwolf
Browse files

strip decoded outputs

parent 01a3966b
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
pytorch_pretrained_bert/tokenization_openai.py
View file @ 850da1cc
...@@ -228,5 +228,5 @@ class OpenAIGPTTokenizer(object): ...@@ -228,5 +228,5 @@ class OpenAIGPTTokenizer(object):
def decode(self, ids, skip_special_tokens=False): def decode(self, ids, skip_special_tokens=False):
"""Converts a sequence of ids in a string.""" """Converts a sequence of ids in a string."""
tokens = self.convert_ids_to_tokens(ids, skip_special_tokens=skip_special_tokens) tokens = self.convert_ids_to_tokens(ids, skip_special_tokens=skip_special_tokens)
out_string = ''.join(tokens).replace('</w>', ' ') out_string = ''.join(tokens).replace('</w>', ' ').strip()
return out_string return out_string
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment