Fixes to support subscriptions with other VMs (#105)

* Update scripts to handle cases where you have other VMs in your sub

* Support subs with other VMs and fix for PDSH permission error

* Minor fix to support subs with other VMs

azure/azure_ssh.sh

 #!/bin/bash
 
 config_file=azure_config.json
+if [ ! -f ${config_file} ]; then
+    echo "Cannot find $config_file"
+    exit 1
+fi
+
+location=`cat ${config_file} | jq .location | sed 's/"//g'`
+rg=deepspeed_rg_$location
+
 while getopts 'c:' flag; do
   case "${flag}" in
     c) config_file="${OPTARG}" ;;
@@ -13,7 +21,7 @@ echo "Using $config_file"
 nodeid=$1
 cmds=${@:2}
 echo $nodeid $cmds
-ip_addr=`az vm list-ip-addresses | jq .[${nodeid}].virtualMachine.network.publicIpAddresses[0].ipAddress | sed 's/"//g'`
+ip_addr=`az vm list-ip-addresses -g $rg | jq .[${nodeid}].virtualMachine.network.publicIpAddresses[0].ipAddress | sed 's/"//g'`
 
 ssh_private_key=`cat ${config_file} | jq .ssh_private_key | sed 's/"//g'`
 if [ $ssh_private_key == "null" ]; then echo 'missing ssh_private_key in config'; exit 1; fi

azure/setup_docker.sh

@@ -5,6 +5,8 @@ if [ ! -f ${azure_config} ]; then
     echo "Cannot find $azure_config"
     exit 1
 fi
+location=`cat ${azure_config} | jq .location | sed 's/"//g'`
+rg=deepspeed_rg_$location
 
 parallel=true
 command -v pdsh
@@ -34,14 +36,14 @@ if [ $parallel == true ]; then
     echo "parallel docker pull"
     hosts=""
     for node_id in {0..1}; do
-        addr=`az vm list-ip-addresses | jq .[${node_id}].virtualMachine.network.publicIpAddresses[0].ipAddress | sed 's/"//g'`
+        addr=`az vm list-ip-addresses  -g $rg | jq .[${node_id}].virtualMachine.network.publicIpAddresses[0].ipAddress | sed 's/"//g'`
         hosts="${addr},${hosts}"
     done
-    PDSH_SSH_ARGS_APPEND=${args} pdsh -w $hosts -l ${username} $update_script
+     PDSH_RCMD_TYPE=ssh  PDSH_SSH_ARGS_APPEND=${args} pdsh -w $hosts -l ${username} $update_script
 else
     echo "sequential docker pull"
     for node_id in `seq 0 $((num_vms - 1))`; do
-        ip_addr=`az vm list-ip-addresses | jq .[${node_id}].virtualMachine.network.publicIpAddresses[0].ipAddress | sed 's/"//g'`
+        ip_addr=`az vm list-ip-addresses  -g $rg | jq .[${node_id}].virtualMachine.network.publicIpAddresses[0].ipAddress | sed 's/"//g'`
         addr=${username}@${ip_addr}
         ssh ${args} $addr $update_script
     done

azure/setup_vms.sh

@@ -5,6 +5,8 @@ if [ ! -f ${azure_config} ]; then
     echo "Cannot find $azure_config"
     exit 1
 fi
+location=`cat ${azure_config} | jq .location | sed 's/"//g'`
+rg=deepspeed_rg_$location
 
 ssh_key=`cat ${azure_config} | jq .ssh_private_key | sed 's/"//g'`
 if [ $ssh_key == "null" ]; then echo 'missing ssh_private_key in config'; exit 1; fi
@@ -14,8 +16,8 @@ if [ $docker_ssh_port == "null" ]; then echo 'missing docker_ssh_port in config'
 username=deepspeed
 args="-i ${ssh_key} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
 
-num_vms=`az vm list | jq '. | length'`
-first_ip_addr=`az vm list-ip-addresses | jq .[0].virtualMachine.network.publicIpAddresses[0].ipAddress | sed 's/"//g'`
+num_vms=`az vm list  -g $rg | jq '. | length'`
+first_ip_addr=`az vm list-ip-addresses  -g $rg | jq .[0].virtualMachine.network.publicIpAddresses[0].ipAddress | sed 's/"//g'`
 num_slots=`ssh $args ${username}@${first_ip_addr} 'nvidia-smi -L | wc -l'`
 echo "number of slots per vm: $num_slots"
 
@@ -24,7 +26,7 @@ ssh_config=config
 echo -n "" > $hostfile
 echo -n "" > $ssh_config
 for node_id in `seq 0 $((num_vms - 1))`; do
-    private_ip_addr=`az vm list-ip-addresses | jq .[${node_id}].virtualMachine.network.privateIpAddresses[0] | sed 's/"//g'`
+    private_ip_addr=`az vm list-ip-addresses  -g $rg | jq .[${node_id}].virtualMachine.network.privateIpAddresses[0] | sed 's/"//g'`
     echo "worker-${node_id} slots=${num_slots}" >> hostfile
     echo "Host worker-${node_id}
     HostName ${private_ip_addr}
@@ -41,7 +43,7 @@ git clone https://github.com/microsoft/DeepSpeed.git workdir/DeepSpeed;
 "
 
 for node_id in `seq 0 $((num_vms - 1))`; do
-    ip_addr=`az vm list-ip-addresses | jq .[${node_id}].virtualMachine.network.publicIpAddresses[0].ipAddress | sed 's/"//g'`
+    ip_addr=`az vm list-ip-addresses  -g $rg | jq .[${node_id}].virtualMachine.network.publicIpAddresses[0].ipAddress | sed 's/"//g'`
     addr=${username}@${ip_addr}
     echo "copying ssh keys, ssh config, hostfile to worker-${node_id}"
     ssh $args ${addr} $update_script