Commit 2ba1a14f authored by LysandreJik's avatar LysandreJik
Browse files

Decode now calls private property instead of public method

parent e00b4ff1
...@@ -636,9 +636,9 @@ class PreTrainedTokenizer(object): ...@@ -636,9 +636,9 @@ class PreTrainedTokenizer(object):
filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens) filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)
text = self.convert_tokens_to_string(filtered_tokens) text = self.convert_tokens_to_string(filtered_tokens)
if self.sep_token is not None and self.sep_token in text: if self._sep_token is not None and self._sep_token in text:
text = text.replace(self.cls_token, self.sep_token) text = text.replace(self._cls_token, self._sep_token)
split_text = list(filter(lambda sentence: len(sentence) > 0, text.split(self.sep_token))) split_text = list(filter(lambda sentence: len(sentence) > 0, text.split(self._sep_token)))
if clean_up_tokenization_spaces: if clean_up_tokenization_spaces:
clean_text = [self.clean_up_tokenization(text) for text in split_text] clean_text = [self.clean_up_tokenization(text) for text in split_text]
return clean_text return clean_text
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment