Merge pull request #67 from UnicornChan/main

[fix] fix bugs about Qwen2-57B, install requirement, DockerFile

.github/workflows/package_wheel_release.yml

@@ -28,7 +28,6 @@ jobs:
          - { os: ubuntu-20.04, pyver:  '3.12', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'}
          - { os: ubuntu-20.04, pyver:  '3.12', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'}
          - { os: ubuntu-20.04, pyver:  '3.12', cuda: '12.1.1', torch: '2.4.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'}
-         - { os: ubuntu-20.04, pyver:  '3.12', cuda: '12.5.1', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'}
          - { os: ubuntu-20.04, pyver:  '3.12', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'}
          - { os: ubuntu-20.04, pyver:  '3.12', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX512', torch_cu: '121'}
          - { os: ubuntu-20.04, pyver:  '3.12', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.0;8.6;8.7;8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'}
@@ -187,8 +186,10 @@ jobs:
             }
           }
           if ($IsWindows) {
+            if (Test-Path -Path "$env:CUDA_PATH/Library/bin/nvcc.exe"){
               $env:CUDA_PATH = "$env:CUDA_PATH/Library"
               $env:CUDA_HOME = $env:CUDA_PATH
+            }
             $env:PATH = "$env:CUDA_PATH/bin;" + $env:PATH
             $directory = "$env:CUDA_PATH/lib/x64/"
             if (-not (Test-Path -Path $directory)) {
@@ -197,7 +198,7 @@ jobs:
             }
             cp $env:CUDA_PATH/lib/*.lib $env:CUDA_PATH/lib/x64/
             $env:INCLUDE =$env:CUDA_PATH + "/include/targets/x64;" + $env:INCLUDE
-
+            $env:INCLUDE =$env:CONDA_PREFIX + "/include;" + $env:INCLUDE
           }
           python -m pip install torch==${{ matrix.torch }} torchvision torchaudio --index-url https://download.pytorch.org/whl/cu${{ matrix.torch_cu }}
           python -m pip install cpufeature build wheel ninja packaging setuptools

.github/workflows/package_wheel_test.yml

-name: Build Wheels
+name: Build Wheels Tests
 on: 
   workflow_dispatch:
     inputs:
@@ -16,8 +16,10 @@ jobs:
       matrix:
         include:
         # Ubuntu
+        - { os: ubuntu-20.04, pyver:  '3.12', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '124'}
         - { os: ubuntu-20.04, pyver:  '3.12', cuda: '12.2.2', torch: '2.3.0', cudaarch: '8.9;9.0+PTX', instruct: 'FANCY', torch_cu: '121'}
         - { os: windows-2022, pyver:  '3.11', cuda: '12.5.1', torch: '2.4.0', cudaarch: '8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '124'}
+        - { os: windows-2022, pyver:  '3.12', cuda: '12.1.1', torch: '2.3.0', cudaarch: '8.9;9.0+PTX', instruct: 'AVX2', torch_cu: '121'}
 
     defaults:
       run:
@@ -96,12 +98,19 @@ jobs:
             }
           }
           if ($IsWindows) {
+            if (Test-Path -Path "$env:CUDA_PATH/Library/bin/nvcc.exe"){
               $env:CUDA_PATH = "$env:CUDA_PATH/Library"
               $env:CUDA_HOME = $env:CUDA_PATH
+            }
             $env:PATH = "$env:CUDA_PATH/bin;" + $env:PATH
+            $directory = "$env:CUDA_PATH/lib/x64/"
+            if (-not (Test-Path -Path $directory)) {
+              New-Item -ItemType Directory -Path $directory
+              Write-Output "Directory '$directory' created."
+            }
             cp $env:CUDA_PATH/lib/*.lib $env:CUDA_PATH/lib/x64/
             $env:INCLUDE =$env:CUDA_PATH + "/include/targets/x64;" + $env:INCLUDE
-
+            $env:INCLUDE =$env:CONDA_PREFIX + "/include;" + $env:INCLUDE
           }
           python -m pip install torch==${{ matrix.torch }} torchvision torchaudio --index-url https://download.pytorch.org/whl/cu${{ matrix.torch_cu }}
           python -m pip install cpufeature build wheel ninja packaging setuptools

Dockerfile

@@ -12,6 +12,7 @@ EOF
 
 FROM pytorch/pytorch:2.3.1-cuda12.1-cudnn8-devel as compile_server
 WORKDIR /workspace
+ENV CUDA_HOME /usr/local/cuda
 COPY --from=web_compile /home/ktransformers /workspace/ktransformers
 RUN <<EOF
 apt update -y &&  apt install -y  --no-install-recommends \
@@ -27,7 +28,7 @@ git submodule init &&
 git submodule update &&
 pip install ninja pyproject numpy cpufeature &&
 pip install flash-attn &&
-CPU_INSTRUCT=NATIVE  KTRANSFORMERS_FORCE_BUILD=TRUE TORCH_CUDA_ARCH_LIST="8.0;8.6;8.7;8.9" pip install . --no-build-isolation --verbose &&
+CPU_INSTRUCT=NATIVE  KTRANSFORMERS_FORCE_BUILD=TRUE TORCH_CUDA_ARCH_LIST="8.0;8.6;8.7;8.9;9.0+PTX" pip install . --no-build-isolation --verbose &&
 pip cache purge
 EOF
 

ktransformers/optimize/optimize_rules/Qwen2-57B-A14B-Instruct.yaml

-- match:
-    name: "^model\\.layers\\..*\\."
-  replace:
-    class: "default"
-    kwargs:
-      generate_device: "cuda"
-      prefill_device: "cuda"
 - match:
     class: ktransformers.models.modeling_qwen2_moe.Qwen2MoeRotaryEmbedding
   replace:

pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
   "torch >= 2.3.0",
   "transformers == 4.43.2",
   "fastapi >= 0.111.0",
+  "uvicorn >= 0.30.1",
   "langchain >= 0.2.0",
   "blessed >= 1.20.0",
   "accelerate >= 0.31.0",