run-7b-sft-lora.sh

#!/bin/bash
#SBATCH -p kshdexclu11
#SBATCH -N 4
#SBATCH --cpus-per-task=1
#SBATCH --ntasks-per-node=32
#SBATCH --gres=dcu:4
#SBATCH -J baichuan
#SBATCH -o logs-13B/baichuan-lora-%j.out
#SBATCH -e logs-13B/baichuan-lora-%j.err
#SBATCH --exclusive

ulimit -s unlimited
export HIP_VISIBLE_DEVICES=0,1,2,3 
export MIOPEN_FIND_MODE=3
export MIOPEN_DEBUG_CONV_IMPLICIT_GEMM=0 
export MIOPEN_USER_DB_PATH=/tmp/miopen-udb
export MIOPEN_CUSTOM_CACHE_DIR=/tmp/miopen-cache
export NCCL_SOCKET_IFNAME=ib0
export HSA_FORCE_FINE_GRAIN_PCIE=1
export OMP_NUM_THREADS=1
export NCCL_IB_HCA=mlx5


export NCCL_DEBUG=INFO
export MIOPEN_COMPILE_PARALLEL_LEVEL=1
export NCCL_PLUGIN_P2P=ucx
export NCCL_P2P_LEVEL=5

echo "START TIME: $(date)"

nodes=($(scontrol show hostnames $SLURM_JOB_NODELIST ))
nodes_array=($nodes)
head_node=${nodes_array[0]}
head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address)
echo Node IP: $head_node_ip 
echo headnode: $head_node 
NODE_RANK=$SLURM_NODEID

hostfile=./hostfile/$SLURM_JOB_ID #获取节点号
scontrol show hostnames $SLURM_JOB_NODELIST > ${hostfile}
rm `pwd`/hostfile-dl -f


for i in `cat $hostfile`
do
    echo ${i} slots=4 >> `pwd`/hostfile/hostfile-dl-$SLURM_JOB_ID #节点号
done
np=$(cat $hostfile|sort|uniq |wc -l) #节点去重
np=$(($np*4))
nodename=$(cat $hostfile |sed -n "1p") #读取每行节点 第一个是主节点
dist_url=`echo $nodename | awk '{print $1}'`

mpirun -np $np --allow-run-as-root --hostfile hostfile/hostfile-dl-$SLURM_JOB_ID --bind-to none `pwd`/run-7b-single-lora.sh