Skip to content

GitLab

Unverified Commit b8d0e81f authored by Rick Ho's avatar Rick Ho Committed by GitHub
Browse files

Merge pull request #144 from Fragile-azalea/master 

remove synchronize
parents 9303ac18 6ebec59c
Show whitespace changes
Inline Side-by-side
Showing with 0 additions and 1 deletion
fmoe/distributed.py
View file @ b8d0e81f
...@@ -68,7 +68,6 @@ class DistributedGroupedDataParallel(nn.Module): ...@@ -68,7 +68,6 @@ class DistributedGroupedDataParallel(nn.Module):
if not no_scale and not reduce_after: if not no_scale and not reduce_after:
coalesced /= comm.size() coalesced /= comm.size()
torch.distributed.all_reduce(coalesced, group=comm) torch.distributed.all_reduce(coalesced, group=comm)
torch.cuda.synchronize()
if not no_scale and reduce_after: if not no_scale and reduce_after:
coalesced /= comm.size() coalesced /= comm.size()
synced = _unflatten_dense_tensors(coalesced, grads) synced = _unflatten_dense_tensors(coalesced, grads)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment