Update fan out flag for pdsh (#315)

* update fan out flag for pdsh

deepspeed/pt/deepspeed_run.py

@@ -21,6 +21,7 @@ DLTS_HOSTFILE = "/job/hostfile"
 EXPORT_ENVS = ["NCCL", "PYTHON"]
 DEEPSPEED_ENVIRONMENT_NAME = ".deepspeed_env"
 DEEPSPEED_ENVIRONMENT_PATHS = [os.path.expanduser("~"), '.']
+PDSH_MAX_FAN_OUT = 1024
 
 
 def parse_args(args=None):
@@ -294,7 +295,9 @@ def main(args=None):
         active_workers = ",".join(active_resources.keys())
         logger.info("Running on the following workers: %s" % active_workers)
 
-        pdsh_cmd_args = ['pdsh', '-w', active_workers]
+        # PDSH flags for max node fan out and specific hosts to launch on
+        # See https://linux.die.net/man/1/pdsh for flag details
+        pdsh_cmd_args = ['pdsh', '-f', str(PDSH_MAX_FAN_OUT), '-w', active_workers]
 
         num_nodes = len(active_resources.keys())
         num_gpus_per_node = None