Unverified Commit d03059a6 authored by Satpal Singh Rathore's avatar Satpal Singh Rathore Committed by GitHub
Browse files

Update utils.py

parent 2b699d44
......@@ -84,7 +84,7 @@ def average_losses_across_data_parallel_group(losses):
[loss.clone().detach().view(1) for loss in losses])
torch.distributed.all_reduce(averaged_losses,
group=mpu.get_data_parallel_group())
averaged_losses = averaged_losses / \
averaged_losses = averaged_losses /
torch.distributed.get_world_size(group=mpu.get_data_parallel_group())
return averaged_losses
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment