run_2nodes.sh

for para in $*
do
    if [[ $para == --profiling* ]];then
        profiling=${para#*=}
        export GPU_FLUSH_ON_EXECUTION=1
        export HIP_DIRECT_DISPATCH=0
    fi
done

mpirun -np 16 --hostfile 2nodes \
              --allow-run-as-root \
              --bind-to none \
              --mca plm_rsh_no_tree_spawn 1 \
              --mca plm_rsh_args "-p 12333" \
              --mca btl_tcp_if_include ibs8 \
              train_mixtral_8x7B_2nodes.sh 10.16.6.2 --profiling=$profiling > output.log 2>&1

wait

rm -rf CKPT
#rm -rf mixtral_dataset/my-mixtral_text_document