Skip to content

GitLab

"git@developer.sourcefind.cn:renzhc/diffusers_dcu.git" did not exist on "ad40e265156e18964245ed943943fdeb7d8cf61a"
Commit a04c4cf4 authored by Myle Ott's avatar Myle Ott
Browse files

Fix --remove-bpe to strip trailing BPE symbols

parent c53b2ee0
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
fairseq/dictionary.py
View file @ a04c4cf4
...@@ -59,7 +59,7 @@ class Dictionary(object): ...@@ -59,7 +59,7 @@ class Dictionary(object):
sent = ' '.join(token_string(i) for i in tensor if i != self.eos()) sent = ' '.join(token_string(i) for i in tensor if i != self.eos())
if bpe_symbol is not None: if bpe_symbol is not None:
sent = sent.replace(bpe_symbol, '') sent = (sent + ' ').replace(bpe_symbol, '').rstrip()
return sent return sent
def unk_string(self, escape=False): def unk_string(self, escape=False):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment