remove synchronize

6ebec59c · Fragile-azalea · GitHub · 9303ac18 · 6ebec59c
Unverified Commit 6ebec59c authored Feb 13, 2023 by Fragile-azalea Committed by GitHub Feb 13, 2023
Hide whitespace changes
Inline Side-by-side

Showing with 0 additions and 1 deletion

fmoe/distributed.py fmoe/distributed.py +0 -1

No files found.
--- a/fmoe/distributed.py
+++ b/fmoe/distributed.py
@@ -68,7 +68,6 @@ class DistributedGroupedDataParallel(nn.Module):
                if not no_scale and not reduce_after:
                    coalesced /= comm.size()
                torch.distributed.all_reduce(coalesced, group=comm)
-                torch.cuda.synchronize()
                if not no_scale and reduce_after:
                    coalesced /= comm.size()
                synced = _unflatten_dense_tensors(coalesced, grads)