Fix BPE for dual learning

Summary: Fix integration test

Reviewed By: xianxl

Differential Revision: D18040440

fbshipit-source-id: 98c8ab7970d081f17deb54c69aa35669de12d767

fairseq/data/data_utils.py

@@ -234,6 +234,8 @@ def batch_by_size(
 def process_bpe_symbol(sentence: str, bpe_symbol: str):
     if bpe_symbol == 'sentencepiece':
         sentence = sentence.replace(' ', '').replace('\u2581', ' ').strip()
+    elif bpe_symbol == '_EOW':
+        sentence = sentence.replace(' ', '').replace('_EOW', ' ').strip()
     elif bpe_symbol is not None:
         sentence = (sentence + ' ').replace(bpe_symbol, '').rstrip()
     return sentence