Upload New File

2_eastart-env_check&model_download&llm_inference/scripts/run_benchmark.sh

+export HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export ROCBLAS_COMPUTETYPE_FP16R=0
+export HSA_FORCE_FINE_GRAIN_PCIE=1
+export NCCL_LAUNCH_MODE=GROUP
+export NCCL_NCHANNELS_PER_PEER=16
+export NCCL_MAX_NCHANNELS=16
+export NCCL_MIN_NCHANNELS=16
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+export NCCL_P2P_LEVEL=SYS
+export NCCL_NET_GDR_LEVEL=7
+export NCCL_NET_GDR_READ=1
+export RCCL_SDMA_COPY_ENABLE=0
+export SENDRECV_STREAM_WITH_COMPUTE=1
+export LD_LIBRARY_PATH=/usr/local/lib/python3.10/site-packages/torch/lib/:$LD_LIBRARY_PATH
+export ALLREDUCE_STREAM_WITH_COMPUTE=1
+export ALLREDUCE_STREAM_WITH_COMPUTE=1
+export NCCL_MIN_NCHANNELS=16
+export NCCL_MAX_NCHANNELS=16
+export VLLM_PCIE_USE_CUSTOM_ALLREDUCE=1
+
+export VLLM_RPC_TIMEOUT=100000
+
+
+
+#!/bin/bash
+
+# 模型配置文件路径
+MODELS_CONFIG="/workspace/configs/model_to_test.cfg"
+
+# 结果目录
+RESULTS_DIR="/workspace/test/inference_outputs"
+
+# 读取配置文件，跳过注释和空行
+while IFS= read -r line || [[ -n "$line" ]]; do
+    # 跳过注释行和空行
+    if [[ "$line" =~ ^# ]] || [[ -z "$line" ]]; then
+        continue
+    fi
+    
+    # 解析配置行
+    IFS=';' read -ra CONFIG <<< "$line"
+    
+    model_name="${CONFIG[0]}"
+    model_path="${CONFIG[1]}"
+    tp="${CONFIG[2]}"
+    batch="${CONFIG[3]//,/ }"        # 将逗号替换为空格
+    prompt_tokens="${CONFIG[4]//,/ }"
+    completion_tokens="${CONFIG[5]//,/ }"
+    dtype="${CONFIG[6]}"
+    max_model_len="${CONFIG[7]}"
+    gpu_memory_utilization="${CONFIG[8]}"
+    
+    echo "开始测试模型: $model_name"
+    echo "模型路径: $model_path"
+    echo "参数配置:"
+    echo "  tensor_parallel_size: $tp"
+    echo "  batch_sizes: $batch"
+    echo "  prompt_tokens: $prompt_tokens"
+    echo "  completion_tokens: $completion_tokens"
+    echo "  dtype: $dtype"
+    echo "  max_model_len: $max_model_len"
+    echo "  gpu_memory_utilization: $gpu_memory_utilization"
+    
+    # 创建模型专属结果目录
+    model_result_dir="${RESULTS_DIR}/${model_name}"
+    mkdir -p "$model_result_dir"
+    
+    # 运行基准测试
+    python /workspace/scripts/benchmark_throughput.py \
+        --model "$model_path" \
+        --tensor-parallel-size "$tp" \
+        --num-prompts $batch \
+        --input-len $prompt_tokens \
+        --output-len $completion_tokens \
+        --dtype "$dtype" \
+        --trust-remote-code \
+        --max-model-len "$max_model_len" \
+        --gpu-memory-utilization "$gpu_memory_utilization" \
+        --output-json "${model_result_dir}/${model_name}_tp${tp}.txt" \
+        2>&1 | tee "${model_result_dir}/${model_name}_tp${tp}.log"
+    
+    echo "完成测试模型: $model_name"
+    echo "结果保存在: $model_result_dir"
+    echo "----------------------------------------"
+done < "$MODELS_CONFIG"
\ No newline at end of file