Add barrier() after init_process_group() (#5475)

references/classification/utils.py

@@ -274,6 +274,7 @@ def init_distributed_mode(args):
     torch.distributed.init_process_group(
         backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank
     )
+    torch.distributed.barrier()
     setup_for_distributed(args.rank == 0)
 
 

references/optical_flow/utils.py

@@ -267,6 +267,7 @@ def setup_ddp(args):
         world_size=args.world_size,
         init_method=args.dist_url,
     )
+    torch.distributed.barrier()
 
 
 def reduce_across_processes(val):

references/segmentation/utils.py

@@ -291,4 +291,5 @@ def init_distributed_mode(args):
     torch.distributed.init_process_group(
         backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank
     )
+    torch.distributed.barrier()
     setup_for_distributed(args.rank == 0)

references/video_classification/utils.py

@@ -250,4 +250,5 @@ def init_distributed_mode(args):
     torch.distributed.init_process_group(
         backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank
     )
+    torch.distributed.barrier()
     setup_for_distributed(args.rank == 0)