run_fp16_multi.sh

#!/bin/bash

module unload compiler/rocm/2.9
module rm compiler/rocm/2.9
module load apps/PyTorch/1.5.0a0/hpcx-2.4.1-gcc-7.3.1-rocm3.3
export MIOPEN_DEBUG_DISABLE_FIND_DB=1

#for multinode
source `pwd`/config_multinode.sh


SLURM_NTASKS_PER_NODE=${SLURM_NTASKS_PER_NODE:-$DGXNGPU}
SLURM_JOB_ID=${SLURM_JOB_ID:-$RANDOM}
echo "Run vars: id $SLURM_JOB_ID gpus $SLURM_NTASKS_PER_NODE mparams $MULTI_NODE"

set -e

# start timing
start=$(date +%s)
start_fmt=$(date +%Y-%m-%d\ %r)
echo "STARTING TIMING RUN AT $start_fmt"

# run benchmark
set -x
NUMEPOCHS=${NUMEPOCHS:-120}
LR=${LR:-"2.5e-3"}

echo "running benchmark"

export DATASET_DIR="/public/software/apps/DeepLearning/Data/COCO2017"

hostfile=./$SLURM_JOB_ID
scontrol show hostnames $SLURM_JOB_NODELIST > ${hostfile}
#rm `pwd`/hostfile-dl -f
cat ${hostfile} > `pwd`/tmp
dist_url=`sed -n '1p' ./tmp`
#echo $dist_url

rank=0
num_lines=`cat ./tmp |wc -l`

for((i=0;i<$num_lines-1;i++))
do
   ((rank=$i+1))
    nodename=$(cat ./tmp |sed -n "${rank}p")
    ssh ${nodename} "cd `pwd` && export MIOPEN_DEBUG_DISABLE_FIND_DB=1 && module rm compiler/rocm/2.9 && source /public/home/aiss/Pytorch/env_rocm3.3_torch1.6.sh && python3 -m bind_launch --nnodes=$num_lines --node_rank=$i  --master_addr=${dist_url} --master_port=4567 --nsockets_per_node ${DGXNSOCKET} --ncores_per_socket ${DGXSOCKETCORES} --nproc_per_node $SLURM_NTASKS_PER_NODE --no_hyperthreads --no_membind `pwd`/train_gov_apexfp16.py --epochs ${NUMEPOCHS} --warmup-factor 0 --threshold=0.23 --data ${DATASET_DIR} --batch-size 90 --lr=$LR --warmup 11 --snapshot_path=`pwd`/$dist_url --opt-level O1 --loss-scale=dynamic" &
done 

((i=$num_lines-1))
nodename=$(cat ./tmp |sed -n "${num_lines}p")


ssh ${nodename} "cd `pwd` && export MIOPEN_DEBUG_DISABLE_FIND_DB=1 && module rm compiler/rocm/2.9 && source /public/home/aiss/Pytorch/env_rocm3.3_torch1.6.sh && HIP_VISIBLE_DEVICES=0,1,2,3 python3 -m bind_launch  --nnodes=$num_lines --node_rank=$i  --master_addr=${dist_url} --master_port=4567 --nsockets_per_node ${DGXNSOCKET} --ncores_per_socket ${DGXSOCKETCORES} --nproc_per_node $SLURM_NTASKS_PER_NODE --no_hyperthreads --no_membind `pwd`/train_gov_apexfp16.py --epochs ${NUMEPOCHS} --warmup-factor 0 --threshold=0.23 --data ${DATASET_DIR} --batch-size 90 --lr=$LR --warmup 11 --snapshot_path=`pwd`/$dist_url --opt-level O1 --loss-scale=dynamic"

set +x

sleep 3

# end timing
end=$(date +%s)
end_fmt=$(date +%Y-%m-%d\ %r)
echo "ENDING TIMING RUN AT $end_fmt"

# report result
result=$(( $end - $start ))
result_name="OBJECT_DETECTION"

echo "RESULT,$result_name,,$result,nvidia,$start_fmt"