only global rank 0 can log tensorboard data; avoid multi gpu/node race for the log directory (#296)

deepspeed/pt/deepspeed_light.py

@@ -144,12 +144,12 @@ class DeepSpeedLight(Module):
         self._configure_with_arguments(args, mpu)
         self._do_sanity_check()
 
+        self._init_distributed(dist_init_required)
+
         self.sample_count = 0
-        if self.tensorboard_enabled():
+        if self.tensorboard_enabled() and self.global_rank == 0:
             self.summary_writer = self.get_summary_writer()
 
-        self._init_distributed(dist_init_required)
-
         # Configure distributed model
         self._configure_distributed_model(model)