Fix rearranging of encoder_out in SequenceGenerator

Summary: Pull Request resolved: https://github.com/fairinternal/fairseq-py/pull/625

Differential Revision: D15595787

Pulled By: myleott

fbshipit-source-id: ba6edf305ed41be392194f492e034dd66d1743fe

fairseq/sequence_generator.py

@@ -304,7 +304,7 @@ class SequenceGenerator(object):
                     corr = batch_idxs - torch.arange(batch_idxs.numel()).type_as(batch_idxs)
                     reorder_state.view(-1, beam_size).add_(corr.unsqueeze(-1) * beam_size)
                 model.reorder_incremental_state(reorder_state)
-                model.reorder_encoder_out(encoder_outs, reorder_state)
+                encoder_outs = model.reorder_encoder_out(encoder_outs, reorder_state)
 
             lprobs, avg_attn_scores = model.forward_decoder(
                 tokens[:, :step + 1], encoder_outs, temperature=self.temperature,