#!/bin/bash
export NCCL_TOPO_FILE=null
export NCCL_ALGO=Ring
export NCCL_RINGS="N0 0 7 6 5 4 3 2 1 N0|N1 1 2 3 4 5 6 7 0 N1|N2 2 1 0 7 6 5 4 3 N2|N3 3 4 5 6 7 0 1 2 N3|N4 4 3 2 1 0 7 6 5 N4|N5 5 6 7 0 1 2 3 4 N5|N6 6 5 4 3 2 1 0 7 N6|N7 7 0 1 2 3 4 5 6 N7"

export PYTORCH_MIOPEN_SUGGEST_NHWC=1
export MIOPEN_PRECISION_FP32_FP32_FP32_TF32_FP32=1
export MIOPEN_FIND_MODE=1
export ROCBLAS_MATH_MODE=1

# workaround for the segment fault issue from libcuda.so
export LD_LIBRARY_PATH=/usr/local/lib/:/usr/local/lib64/:/opt/mpi/lib:/opt/hwloc/lib:/opt/dtk/dcc/gcvm/lib:/opt/dtk/hip/lib:/opt/dtk/llvm/lib:/opt/dtk/lib:/opt/dtk/lib64:/opt/hyhal/lib:/opt/hyhal/lib64:/opt/dtk/opencl/lib:/opt/dtk-25.04.4/dcc/gcvm/lib:/opt/dtk-25.04.4/hip/lib:/opt/dtk-25.04.4/llvm/lib:/opt/dtk-25.04.4/lib:/opt/dtk-25.04.4/lib64:/opt/hyhal/lib:/opt/hyhal/lib64:/opt/dtk-25.04.4/opencl/lib:/opt/ucx/lib:/opt/dtk/hip/lib:/opt/dtk/llvm/lib:/opt/dtk/lib:/opt/dtk/lib64:/opt/hyhal/lib:/opt/hyhal/lib64:/opt/mpi/lib:/opt/hwloc/lib:

TIME=$(date "+%Y-%m-%d_%H_%M")

MASTER_ADDR=${1:-localhost}
NNODES=${2:-1}
NODE_RANK=${3:-0}
CONFIG=${4:-./configs/pointpillars/pointpillars_hv_fpn_sbn-all_8xb4-2x_nus-3d.py}

bash tools/dist_train_numa.sh $MASTER_ADDR $NNODES $NODE_RANK $CONFIG \
	2>&1 | tee bw1000_pointpillars_${NNODES}nodes_$TIME.log
