dist_train.sh

#!/usr/bin/env bash
#export PYTORCH_MIOPEN_SUGGEST_NHWC=1
#export MIOPEN_FIND_MODE=1

export TORCHINDUCTOR_LAYOUT_OPTIMIZATION=1
export PYTORCH_MIOPEN_SUGGEST_NHWC=1
export TORCHINDUCTOR_COORDINATE_DESCENT_TUNING=1
export MIOPEN_PRECISION_FP32_FP32_FP32_TF32_FP32=1
export LD_LIBRARY_PATH=/root/rocblas-install/lib:$LD_LIBRARY_PATH

CONFIG=$1
GPUS=$2
NNODES=${NNODES:-1}
NODE_RANK=${NODE_RANK:-0}
PORT=${PORT:-29500}
MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"}

PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
torchrun \
    --nnodes=$NNODES \
    --node_rank=$NODE_RANK \
    --master_addr=$MASTER_ADDR \
    --nproc_per_node=$GPUS \
    --master_port=$PORT \
    $(dirname "$0")/train.py \
    $CONFIG \
    --seed 0 \
    --launcher pytorch ${@:3}