Skip to content

GitLab

Commit 0a7b3572 authored by jerrrrry's avatar jerrrrry
Browse files

Update .gitlab-ci.yml

parent 1999d8ba
Hide whitespace changes
Inline Side-by-side
Showing with 69 additions and 58 deletions
.gitlab-ci.yml
View file @ 0a7b3572
# 定义流水线要使用的 Docker 镜像 # 定义流水线要使用的 Docker 镜像
image: image.sourcefind.cn:5000/dcu/admin/base/vllm:0.8.5-ubuntu22.04-dtk25.04.1-rc5-das1.6-py3.10-20250724 image: image.sourcefind.cn:5000/dcu/admin/base/vllm:0.8.5-ubuntu22.04-dtk25.04.1-rc5-das1.6-py3.10-20250724
# 定义流水线的所有阶段
# 只定义一个阶段
stages: stages:
- prepare
- install
- test - test
# === 阶段 1: 准备阶段 === # === 唯一的作业：准备环境并运行基准测试 ===
download_assets: benchmark_sdxl:
stage: prepare stage: test
tags: tags:
- demos - demos
script: script:
- echo "--- 1. Preparing assets ---" - echo "========================================="
- pip install modelscope - echo "Step 1: Checking and downloading assets"
- nohup modelscope download --model "AI-ModelScope/sdxl-vae-fp16-fix" --local_dir "./sdxl-vae-fp16-fix" > downloadsdxl-vae-fp16-fix.log 2>&1 & - echo "========================================="
- nohup modelscope download --model "stabilityai/stable-diffusion-xl-base-1.0" --local_dir "./stable-diffusion-xl-base-1.0" > downloadsdxl-stable-diffusion-xl-base-1.0.log 2>&1 & # 条件性下载模型
- curl -f -C - -o "rocblas-install-0626-bug.tar.gz" "https://wuzh01.hpccube.com:65015/efile/s/d/amVycnJycnk=/a6a7342d017b1748" - |
- curl -f -C - -o "package-miopen-dev-0801-ubuntu20.tar.gz" "https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/9c2334cb9cdb8b39" if [ ! -d "stable-diffusion-xl-base-1.0" ]; then
- echo "--- Asset download finished ---" echo "Models not found in cache, downloading..."
cache: pip install modelscope -q
key: "assets-cache-${CI_COMMIT_REF_SLUG}" modelscope download --model "AI-ModelScope/sdxl-vae-fp16-fix" --local_dir "./sdxl-vae-fp16-fix" > /dev/null
paths: modelscope download --model "stabilityai/stable-diffusion-xl-base-1.0" --local_dir "./stable-diffusion-xl-base-1.0" > /dev/null
- "sdxl-vae-fp16-fix/" else
- "stable-diffusion-xl-base-1.0/" echo "Models found in cache, skipping download."
- "rocblas-install-0626-bug.tar.gz" fi
- "package-miopen-dev-0801-ubuntu20.tar.gz" # 条件性下载库和包
- |
if [ ! -f "diffusers-0.33.1-py3-none-any.whl" ]; then
echo "Libraries/packages not found in cache, downloading..."
curl -s -f -C - -o "rocblas-install-0626-bug.tar.gz" "https://wuzh01.hpccube.com:65015/efile/s/d/amVycnJycnk=/a6a7342d017b1748"
curl -s -f -C - -o "package-miopen-dev-0801-ubuntu20.tar.gz" "https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/9c2334cb9cdb8b39"
curl -s -f -C - -o "diffusers-0.33.1-py3-none-any.whl" "https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/63ec0d10ce960f90"
curl -s -f -C - -o "lightop-0.5.0+das.dtk25041-cp310-cp310-linux_x86_64.whl" "https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/da522f7e175bb092"
curl -s -f -C - -o "litserve-0.2.15-py3-none-any.whl" "https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/36943fb1ca62ac3b"
curl -s -f -C - -o "transformers.tar.gz" "https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/f587d939a37d1727"
else
echo "Libraries/packages found in cache, skipping download."
fi
# === 阶段 2: 安装阶段 === - echo "========================================="
setup_environment: - echo "Step 2: Setting up the environment"
stage: install - echo "========================================="
tags: # 解压库文件
- demos - tar -xzf "rocblas-install-0626-bug.tar.gz"
script: - tar -xzf "package-miopen-dev-0801-ubuntu20.tar.gz"
- echo "--- 2. Setting up environment ---" - tar -xzf "transformers.tar.gz"
- tar -xzvf "rocblas-install-0626-bug.tar.gz" # 替换 VAE 权重
- tar -xzvf "package-miopen-dev-0801-ubuntu20.tar.gz"
- cd "stable-diffusion-xl-base-1.0" - cd "stable-diffusion-xl-base-1.0"
- mv vae vae_bak || true - mv vae vae_bak || true
- mkdir -p vae - mkdir -p vae
...@@ -44,44 +54,45 @@ setup_environment: ...@@ -44,44 +54,45 @@ setup_environment:
- cp ../sdxl-vae-fp16-fix/s* ./vae/ - cp ../sdxl-vae-fp16-fix/s* ./vae/
- cp ../sdxl-vae-fp16-fix/d* ./vae/ - cp ../sdxl-vae-fp16-fix/d* ./vae/
- cd .. - cd ..
- curl -f -C - -o "diffusers-0.33.1-py3-none-any.whl" "https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/63ec0d10ce960f90"
- curl -f -C - -o "lightop-0.5.0+das.dtk25041-cp310-cp310-linux_x86_64.whl" "https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/da522f7e175bb092" - echo "========================================="
- curl -f -C - -o "litserve-0.2.15-py3-none-any.whl" "https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/36943fb1ca62ac3b" - echo "Step 3: Installing Python packages"
- pip install "diffusers-0.33.1-py3-none-any.whl" "lightop-0.5.0+das.dtk25041-cp310-cp310-linux_x86_64.whl" "litserve-0.2.15-py3-none-any.whl" - echo "========================================="
- curl -f -C - -o "transformers.tar.gz" "https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/f587d939a37d1727" # 安装 Python 包 (有缓存时会非常快)
- tar -xzvf "transformers.tar.gz" - pip install "diffusers-0.33.1-py3-none-any.whl" "lightop-0.5.0+das.dtk25041-cp310-cp310-linux_x86_64.whl" "litserve-0.2.15-py3-none-any.whl" -q
- echo "--- Environment setup finished ---"
- echo "========================================="
- echo "Step 4: Running the benchmark"
- echo "========================================="
# 设置环境变量并运行测试
- export LD_LIBRARY_PATH="$CI_PROJECT_DIR/rocblas-install/lib/:$LD_LIBRARY_PATH"
- export LD_LIBRARY_PATH="$CI_PROJECT_DIR/package-miopen-dev-0801-ubuntu20/lib/:$LD_LIBRARY_PATH"
- export PYTORCH_MIOPEN_SUGGEST_NHWC=1
- export PYTHONPATH="$CI_PROJECT_DIR/transformers:$PYTHONPATH"
- export MODEL_PATH="$CI_PROJECT_DIR/stable-diffusion-xl-base-1.0"
- python test.py > benchmark.log 2>&1
- echo "--- Benchmark finished, showing last 20 lines of log ---"
- tail -n 20 benchmark.log
# 定义缓存，拉取并更新
cache: cache:
key: "assets-cache-${CI_COMMIT_REF_SLUG}" key: "sdxl-all-in-one-cache-${CI_COMMIT_REF_SLUG}"
paths: paths:
- "sdxl-vae-fp16-fix/" - "sdxl-vae-fp16-fix/"
- "stable-diffusion-xl-base-1.0/" - "stable-diffusion-xl-base-1.0/"
- "rocblas-install/" - "rocblas-install/"
- "package-miopen-dev-0801-ubuntu20/" - "package-miopen-dev-0801-ubuntu20/"
- "transformers/" - "transformers/"
policy: pull-push - "diffusers-0.33.1-py3-none-any.whl"
- "lightop-0.5.0+das.dtk25041-cp310-cp310-linux_x86_64.whl"
# === 阶段 3: 测试阶段 === - "litserve-0.2.15-py3-none-any.whl"
benchmark_sdxl: - "transformers.tar.gz"
stage: test - "/root/.cache/pip"
tags: # 定义产物
- demos
script:
- echo "--- 3. Running SDXL benchmark ---"
# 设置环境变量
- export LD_LIBRARY_PATH="$CI_PROJECT_DIR/rocblas-install/lib/:$LD_LIBRARY_PATH"
- export LD_LIBRARY_PATH="$CI_PROJECT_DIR/package-miopen-dev-0801-ubuntu20/lib/:$LD_LIBRARY_PATH"
- export PYTORCH_MIOPEN_SUGGEST_NHWC=1
- export PYTHONPATH="$CI_PROJECT_DIR/transformers:$PYTHONPATH"
- export MODEL_PATH="$CI_PROJECT_DIR/stable-diffusion-xl-base-1.0"
- echo "Starting benchmark script, full log will be saved to benchmark.log..."
# 将 Python 脚本的所有输出（包括错误）都重定向到 benchmark.log 文件
- python test.py
artifacts: artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
paths: paths:
- "*.png" - "*.png"
- "results.json"
- "benchmark.log"
expire_in: 1 week expire_in: 1 week
cache:
key: "assets-cache-${CI_COMMIT_REF_SLUG}"
policy: pull
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment