Upload New File

084-servering/test.sh

+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export HSA_FORCE_FINE_GRAIN_PCIE=1
+export NCCL_MAX_NCHANNELS=16
+export NCCL_MIN_NCHANNELS=16
+export NCCL_P2P_LEVEL=SYS
+export NCCL_LAUNCH_MODE=GROUP
+export ROCBLAS_COMPUTETYPE_FP16R=0
+export LD_LIBRARY_PATH=/usr/local/lib/python3.10/site-packages/torch/lib/:$LD_LIBRARY_PATH
+export VLLM_NUMA_BIND=1
+export VLLM_RANK0_NUMA=3
+export VLLM_RANK1_NUMA=1
+export VLLM_RANK2_NUMA=1
+export VLLM_RANK3_NUMA=0
+export VLLM_RANK4_NUMA=7
+export VLLM_RANK5_NUMA=5
+export VLLM_RANK6_NUMA=5
+export VLLM_RANK7_NUMA=4
+echo "tp,data_type,batch,prompt_tokens,completion_tokens,TOTAL_THROUGHPUT(toks/s),generate_throughput(toks/s), TTFT(ms),TPOT(ms),ITL(ms)" > V3-BF16-085.csv
+pairs=( "4096 512" "8192 512" "16384 512" "20480 512" )
+model_path="/workspace/llms/bf16_model/"
+tp=32
+data_type="BF16"
+mkdir -p ./log/
+for batch in 1 2 4 6 8 10 16 20; do
+    for pair in "${pairs[@]}"; do
+        prompt_tokens=${pair%% *}
+        completion_tokens=${pair#* }
+        echo "data_type: $data_type,batch: $batch, prompt_tokens: $prompt_tokens, completion_tokens: $completion_tokens, tp: ${tp}"
+        log_path="log/vllm_${model}_batch_${batch}_prompt_tokens_${prompt_tokens}_completion_tokens_${completion_tokens}_tp_${tp}.log"
+        touch $log_path
+        # benchmark_throughput.py
+        python benchmark_serving.py \
+                --backend openai \
+                --port 8000 \
+                --model ${model_path} \
+                --trust-remote-code \
+                --dataset-name random \
+                --ignore-eos \
+                --random-input-len ${prompt_tokens} \
+                --random-output-len ${completion_tokens} \
+                --num-prompts ${batch}  \
+                2>&1 | tee  $log_path
+        #metric
+        E2E_TIME=`grep "^Benchmark duration" $log_path | awk -F ' ' '{print $4}'`
+        REQ_THROUGHPUT=`grep "^Request throughput"  $log_path| awk -F ' ' '{print $4}'`
+        GEN_THROUGHPUT=`grep "^Output token"  $log_path| awk -F ' ' '{print $5}'`
+        TOTAL_THROUGHPUT=`grep "^Total Token" $log_path| awk -F ' ' '{print $5}'`
+        TTFT=`grep "^Mean TTFT"  $log_path| awk -F ' ' '{print $4}'`
+        TPOT=`grep "^Mean TPOT"  $log_path| awk -F ' ' '{print $4}'`
+        ITL=`grep "^Mean ITL"  $log_path| awk -F ' ' '{print $4}'`
+        P99_ITL=`grep "^P99 ITL"  $log_path| awk -F ' ' '{print $4}'`
+        P99_TTFT=`grep "^P99 TTFT"  $log_path| awk -F ' ' '{print $4}'`
+        P99_TPOT=`grep "^P99 TPOT"  $log_path| awk -F ' ' '{print $4}'`
+        echo "$tp,$data_type,$batch,$prompt_tokens,$completion_tokens,$TOTAL_THROUGHPUT,$GEN_THROUGHPUT,$TTFT,$TPOT, $ITL" >> V3-BF16-085.csv
+    done
+done
\ No newline at end of file