run.sh

#!/bin/bash

#for llama2-7b pretrain test
#mpirun -np 8 --allow-run-as-root ./llama2_7b.sh localhost . >& llama2_7b.log &

#for llama2-13b pretrain test
#mpirun -np 8 --allow-run-as-root ./llama2_13b.sh localhost . >& llama2_13b.log &

/opt/mpi/bin/mpirun \
        --allow-run-as-root \
        -np 32 \
        --hostfile hostfile \
        --tag-output \
        --merge-stderr-to-stdout \
        --output-filename tmp \
        -mca plm_rsh_args "-p 1777" \
        --bind-to none \
        -mca plm_rsh_no_tree_spawn 1 \
        ./qwen2_72b.sh \
        node116
#how to calculate tgs(tokens/s/gpu)
#TGS = sequence_length * global_batchsize / elapsed_time_per_iteration(s) / total_gpu_cards