Fix tokenization (fixes #926) (#929)

Summary:
Fixes https://github.com/pytorch/fairseq/issues/926
Pull Request resolved: https://github.com/pytorch/fairseq/pull/929

Differential Revision: D16560281

Pulled By: myleott

fbshipit-source-id: 751051bcdbf25207315bb05f5bee0235d21be627

fairseq/models/roberta/hub_interface.py

@@ -36,8 +36,8 @@ class RobertaHubInterface(nn.Module):
     def encode(self, sentence: str, *addl_sentences) -> torch.LongTensor:
         bpe_sentence = '<s> ' + self.bpe.encode(sentence) + ' </s>'
         for s in addl_sentences:
-            bpe_sentence += ' </s> ' + self.bpe.encode(s)
-        tokens = self.task.source_dictionary.encode_line(bpe_sentence, append_eos=True)
+            bpe_sentence += ' </s> ' + self.bpe.encode(s) + ' </s>'
+        tokens = self.task.source_dictionary.encode_line(bpe_sentence, append_eos=False)
         return tokens.long()
 
     def extract_features(self, tokens: torch.LongTensor, return_all_hiddens=False) -> torch.Tensor: