Commit 45082e48 authored by alexeib's avatar alexeib Committed by Myle Ott
Browse files

make batching faster for monolingual dataset

parent 0b5166db
......@@ -63,8 +63,7 @@ class MonolingualDataset(FairseqDataset):
def num_tokens(self, index):
"""Return an example's length (number of tokens), used for batching."""
source, target = self.dataset[index]
return len(source)
return self.sizes[index]
def ordered_indices(self):
"""Ordered indices for batching."""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment