Commit 04575d01 authored by jerrrrry's avatar jerrrrry
Browse files

Update .gitlab-ci.yml

parent ae055819
# 定义流水线要使用的 Docker 镜像 # 定义流水线要使用的 Docker 镜像
image: image.sourcefind.cn:5000/dcu/admin/base/vllm:0.8.5-ubuntu22.04-dtk25.04.1-rc5-das1.6-py3.10-20250724 image: image.sourcefind.cn:5000/dcu/admin/base/vllm:0.8.5-ubuntu22.04-dtk25.04.1-rc5-das1.6-py3.10-20250724
# 定义流水线的所有阶段 # 定义流水线的所有阶段
stages: stages:
- prepare - prepare
...@@ -14,16 +13,12 @@ download_assets: ...@@ -14,16 +13,12 @@ download_assets:
- demos - demos
script: script:
- echo "--- 1. Preparing assets ---" - echo "--- 1. Preparing assets ---"
# 安装 modelscope
- pip install modelscope - pip install modelscope
# 下载模型
- modelscope download --model AI-ModelScope/sdxl-vae-fp16-fix --local_dir ./sdxl-vae-fp16-fix - modelscope download --model AI-ModelScope/sdxl-vae-fp16-fix --local_dir ./sdxl-vae-fp16-fix
- modelscope download --model stabilityai/stable-diffusion-xl-base-1.0 --local_dir ./stable-diffusion-xl-base-1.0 - modelscope download --model stabilityai/stable-diffusion-xl-base-1.0 --local_dir ./stable-diffusion-xl-base-1.0
# 下载库文件
- curl -f -C - -o rocblas-install-0626-bug.tar.gz https://wuzh01.hpccube.com:65015/efile/s/d/amVycnJycnk=/a6a7342d017b1748 - curl -f -C - -o rocblas-install-0626-bug.tar.gz https://wuzh01.hpccube.com:65015/efile/s/d/amVycnJycnk=/a6a7342d017b1748
- curl -f -C - -o package-miopen-dev-0801-ubuntu20.tar.gz https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/9c2334cb9cdb8b39 - curl -f -C - -o package-miopen-dev-0801-ubuntu20.tar.gz https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/9c2334cb9cdb8b39
- echo "--- Asset download finished ---" - echo "--- Asset download finished ---"
# 使用缓存来保存下载的文件,key为分支名,paths为要缓存的目录
cache: cache:
key: ${CI_COMMIT_REF_SLUG} key: ${CI_COMMIT_REF_SLUG}
paths: paths:
...@@ -39,27 +34,22 @@ setup_environment: ...@@ -39,27 +34,22 @@ setup_environment:
- demos - demos
script: script:
- echo "--- 2. Setting up environment ---" - echo "--- 2. Setting up environment ---"
# 解压库文件
- tar -xzvf rocblas-install-0626-bug.tar.gz - tar -xzvf rocblas-install-0626-bug.tar.gz
- tar -xzvf package-miopen-dev-0801-ubuntu20.tar.gz - tar -xzvf package-miopen-dev-0801-ubuntu20.tar.gz
# 替换 VAE 权重
- cd stable-diffusion-xl-base-1.0 - cd stable-diffusion-xl-base-1.0
- mv vae vae_bak || true # 如果 vae_bak 已存在,忽略错误 - mv vae vae_bak || true
- mkdir -p vae - mkdir -p vae
- cp ../sdxl-vae-fp16-fix/c* ./vae/ - cp ../sdxl-vae-fp16-fix/c* ./vae/
- cp ../sdxl-vae-fp16-fix/s* ./vae/ - cp ../sdxl-vae-fp16-fix/s* ./vae/
- cp ../sdxl-vae-fp16-fix/d* ./vae/ - cp ../sdxl-vae-fp16-fix/d* ./vae/
- cd .. - cd ..
# 下载并安装 Python 包
- curl -f -C - -o diffusers-0.33.1-py3-none-any.whl https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/63ec0d10ce960f90 - curl -f -C - -o diffusers-0.33.1-py3-none-any.whl https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/63ec0d10ce960f90
- curl -f -C - -o lightop-0.5.0+das.dtk25041-cp310-cp310-linux_x86_64.whl https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/da522f7e175bb092 - curl -f -C - -o lightop-0.5.0+das.dtk25041-cp310-cp310-linux_x86_64.whl https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/da522f7e175bb092
- curl -f -C - -o litserve-0.2.15-py3-none-any.whl https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/36943fb1ca62ac3b - curl -f -C - -o litserve-0.2.15-py3-none-any.whl https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/36943fb1ca62ac3b
- pip install diffusers-0.33.1-py3-none-any.whl lightop-0.5.0+das.dtk25041-cp310-cp310-linux_x86_64.whl litserve-0.2.15-py3-none-any.whl - pip install diffusers-0.33.1-py3-none-any.whl lightop-0.5.0+das.dtk25041-cp310-cp310-linux_x86_64.whl litserve-0.2.15-py3-none-any.whl
# 替换 transformers
- curl -f -C - -o transformers.tar.gz https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/f587d939a37d1727 - curl -f -C - -o transformers.tar.gz https://ksefile.hpccube.com:65241/efile/s/d/amVycnJycnk=/f587d939a37d1727
- tar -xzvf transformers.tar.gz - tar -xzvf transformers.tar.gz
- echo "--- Environment setup finished ---" - echo "--- Environment setup finished ---"
# 继承上一个作业的缓存,并缓存 pip 安装的包
cache: cache:
key: ${CI_COMMIT_REF_SLUG} key: ${CI_COMMIT_REF_SLUG}
paths: paths:
...@@ -68,7 +58,6 @@ setup_environment: ...@@ -68,7 +58,6 @@ setup_environment:
- rocblas-install/ - rocblas-install/
- package-miopen-dev-0801-ubuntu20/ - package-miopen-dev-0801-ubuntu20/
- transformers/ - transformers/
# 缓存 pip 的安装目录,加速后续安装
policy: pull-push policy: pull-push
# === 阶段 3: 测试阶段 - 运行基准测试 === # === 阶段 3: 测试阶段 - 运行基准测试 ===
...@@ -76,13 +65,11 @@ benchmark_sdxl: ...@@ -76,13 +65,11 @@ benchmark_sdxl:
stage: test stage: test
tags: tags:
- demos - demos
# 在脚本执行前设置所有必要的环境变量
before_script: before_script:
- export LD_LIBRARY_PATH=$CI_PROJECT_DIR/rocblas-install/lib/:$LD_LIBRARY_PATH - export LD_LIBRARY_PATH=$CI_PROJECT_DIR/rocblas-install/lib/:$LD_LIBRARY_PATH
- export LD_LIBRARY_PATH=$CI_PROJECT_DIR/package-miopen-dev-0801-ubuntu20/lib/:$LD_LIBRARY_PATH - export LD_LIBRARY_PATH=$CI_PROJECT_DIR/package-miopen-dev-0801-ubuntu20/lib/:$LD_LIBRARY_PATH
- export PYTORCH_MIOPEN_SUGGEST_NHWC=1 - export PYTORCH_MIOPEN_SUGGEST_NHWC=1
- export PYTHONPATH="$CI_PROJECT_DIR/transformers:$PYTHONPATH" - export PYTHONPATH="$CI_PROJECT_DIR/transformers:$PYTHONPATH"
# 设置模型路径环境变量,供 test.py 使用
- export MODEL_PATH="$CI_PROJECT_DIR/stable-diffusion-xl-base-1.0" - export MODEL_PATH="$CI_PROJECT_DIR/stable-diffusion-xl-base-1.0"
script: script:
- echo "--- 3. Running SDXL benchmark ---" - echo "--- 3. Running SDXL benchmark ---"
...@@ -90,15 +77,12 @@ benchmark_sdxl: ...@@ -90,15 +77,12 @@ benchmark_sdxl:
- echo "PYTHONPATH: $PYTHONPATH" - echo "PYTHONPATH: $PYTHONPATH"
- python test.py - python test.py
- echo "--- Benchmark finished ---" - echo "--- Benchmark finished ---"
# 定义产物,保存测试生成的图片和性能报告
artifacts: artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
paths: paths:
- "*.png" - "*.png"
- results.json - results.json
# 产物保存 1 周
expire_in: 1 week expire_in: 1 week
# 只从上一个作业继承缓存,不更新
cache: cache:
key: ${CI_COMMIT_REF_SLUG} key: ${CI_COMMIT_REF_SLUG}
policy: pull policy: pull
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment