Upload New File

test.sh.sh

+echo "data_type,batch,prompt_tokens,completion_tokens,tp,all_throughput(tokens/s),generate_throughput(tokens/s),latency(s)" > qwen2.5_14b-tp2.csv
+pairs=("2 1" "2 128" "2 1000"  "2 2000" "512 1" "512 512" "1000 1"  "1000 1000" "2000 1" "2000 2000" "4096 1")
+
+model_path="/workspace/llms/qwen2.5/Qwen2.5-14B-Instruct"
+
+model=${model_path##*/}
+tp=2
+data_type="fp16"
+
+for batch in 1 2 4 8 16 32 64 128; do
+    for pair in "${pairs[@]}"; do
+        prompt_tokens=${pair%% *}
+        completion_tokens=${pair#* }
+i
+        echo "data_type: $data_type,batch: $batch, prompt_tokens: $prompt_tokens, completion_tokens: $completion_tokens, tp: ${tp}"
+
+        # benchmark_throughput.py
+        python ./benchmarks/benchmark_throughput.py --model ${model_path} --tensor-parallel-size ${tp} --num-prompts ${batch} --input-len ${prompt_tokens} --output-len ${completion_tokens} --dtype float16 --trust-remote-code --enforce-eager 2>&1 | tee vllm_${model}_batch_${batch}_prompt_tokens_${prompt_tokens}_completion_tokens_${completion_tokens}_tp_${tp}.log
+    
+        all_tht=`grep "^All Throughput" vllm_${model}_batch_${batch}_prompt_tokens_${prompt_tokens}_completion_tokens_${completion_tokens}_tp_${tp}.log | awk -F ' ' '{print $5}'`
+        gen_tht=`grep "^Generate Throughput" vllm_${model}_batch_${batch}_prompt_tokens_${prompt_tokens}_completion_tokens_${completion_tokens}_tp_${tp}.log | awk -F ' ' '{print $3}'`
+        latency=`grep "^Latency" vllm_${model}_batch_${batch}_prompt_tokens_${prompt_tokens}_completion_tokens_${completion_tokens}_tp_${tp}.log | awk -F ' ' '{print $2}'`
+        
+        if [ -z "$all_tht" ]; then
+            echo "error: not All Throughput"
+            exit 1;
+        elif [ -z "$latency" ]; then
+            echo "error: not Latency"
+            exit 1;
+        fi
+
+        echo "$data_type,$batch,$prompt_tokens,$completion_tokens,$tp,$all_tht,$gen_tht,$latency" >> qwen2.5_14b-tp2.csv
+    done
+done
\ No newline at end of file