removed bcast in local ddp

41c1af0e · Mohammad · d805e1a0 · 41c1af0e
Commit 41c1af0e authored Apr 20, 2020 by Mohammad
Hide whitespace changes
Inline Side-by-side

Showing with 0 additions and 4 deletions

megatron/model/distributed.py megatron/model/distributed.py +0 -4

No files found.
--- a/megatron/model/distributed.py
+++ b/megatron/model/distributed.py
@@ -31,10 +31,6 @@ class DistributedDataParallel(MegatronModule):
        self.module = module
        self.data_parallel_group = mpu.get_data_parallel_group()
-        src_rank = mpu.get_model_parallel_rank()
-        for p in self.module.parameters():
-            if torch.is_tensor(p):
-                dist.broadcast(p, src_rank, group=self.data_parallel_group)
        def allreduce_params(reduce_after=True, no_scale=False, fp32_allreduce=False):
            if(self.needs_reduction):