test.sh

export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export HSA_FORCE_FINE_GRAIN_PCIE=1
export NCCL_MAX_NCHANNELS=16
export NCCL_MIN_NCHANNELS=16
export NCCL_P2P_LEVEL=SYS
export NCCL_LAUNCH_MODE=GROUP
export ROCBLAS_COMPUTETYPE_FP16R=0
export LD_LIBRARY_PATH=/usr/local/lib/python3.10/site-packages/torch/lib/:$LD_LIBRARY_PATH
export VLLM_NUMA_BIND=1
export VLLM_RANK0_NUMA=3
export VLLM_RANK1_NUMA=1
export VLLM_RANK2_NUMA=1
export VLLM_RANK3_NUMA=0
export VLLM_RANK4_NUMA=7
export VLLM_RANK5_NUMA=5
export VLLM_RANK6_NUMA=5
export VLLM_RANK7_NUMA=4
echo "tp,data_type,batch,prompt_tokens,completion_tokens,TOTAL_THROUGHPUT(toks/s),generate_throughput(toks/s), TTFT(ms),TPOT(ms),ITL(ms)" > V3-BF16-085.csv
pairs=( "4096 512" "8192 512" "16384 512" "20480 512" )
model_path="/workspace/llms/bf16_model/"
tp=32
data_type="BF16"
mkdir -p ./log/
for batch in 1 2 4 6 8 10 16 20; do
    for pair in "${pairs[@]}"; do
        prompt_tokens=${pair%% *}
        completion_tokens=${pair#* }
        echo "data_type: $data_type,batch: $batch, prompt_tokens: $prompt_tokens, completion_tokens: $completion_tokens, tp: ${tp}"
        log_path="log/vllm_${model}_batch_${batch}_prompt_tokens_${prompt_tokens}_completion_tokens_${completion_tokens}_tp_${tp}.log"
        touch $log_path
        # benchmark_throughput.py
        python benchmark_serving.py \
                --backend openai \
                --port 8000 \
                --model ${model_path} \
                --trust-remote-code \
                --dataset-name random \
                --ignore-eos \
                --random-input-len ${prompt_tokens} \
                --random-output-len ${completion_tokens} \
                --num-prompts ${batch}  \
                2>&1 | tee  $log_path
        #metric
        E2E_TIME=`grep "^Benchmark duration" $log_path | awk -F ' ' '{print $4}'`
        REQ_THROUGHPUT=`grep "^Request throughput"  $log_path| awk -F ' ' '{print $4}'`
        GEN_THROUGHPUT=`grep "^Output token"  $log_path| awk -F ' ' '{print $5}'`
        TOTAL_THROUGHPUT=`grep "^Total Token" $log_path| awk -F ' ' '{print $5}'`
        TTFT=`grep "^Mean TTFT"  $log_path| awk -F ' ' '{print $4}'`
        TPOT=`grep "^Mean TPOT"  $log_path| awk -F ' ' '{print $4}'`
        ITL=`grep "^Mean ITL"  $log_path| awk -F ' ' '{print $4}'`
        P99_ITL=`grep "^P99 ITL"  $log_path| awk -F ' ' '{print $4}'`
        P99_TTFT=`grep "^P99 TTFT"  $log_path| awk -F ' ' '{print $4}'`
        P99_TPOT=`grep "^P99 TPOT"  $log_path| awk -F ' ' '{print $4}'`
        echo "$tp,$data_type,$batch,$prompt_tokens,$completion_tokens,$TOTAL_THROUGHPUT,$GEN_THROUGHPUT,$TTFT,$TPOT, $ITL" >> V3-BF16-085.csv
    done
done