Make an eval iteration the same number of samples as a training iteration

megatron/training.py

@@ -761,6 +761,7 @@ def evaluate(forward_step_func, data_iterator, model, verbose=False):
                 print_rank_0('Evaluating iter {}/{}'.format(iteration,
                                                             args.eval_iters))
 
+            for _ in range(args.num_microbatches_in_minibatch):
                 if not mpu.is_pipeline_first_stage():
                     input_tensor, _ = communicate(
                         tensor_send_next=None,
@@ -793,7 +794,7 @@ def evaluate(forward_step_func, data_iterator, model, verbose=False):
     model.train()
 
     for key in total_loss_dict:
-        total_loss_dict[key] /= args.eval_iters
+        total_loss_dict[key] /= args.eval_iters * args.num_microbatches_in_minibatch
 
     return total_loss_dict