Skip to content

GitLab

Commit 74f7aa06 authored by limm's avatar limm
Browse files

fix code, compiled and tested successfully

parent 406fb6aa
Show whitespace changes
Inline Side-by-side
Showing with 453 additions and 285 deletions
CMakeLists.txt
View file @ 74f7aa06
......@@ -29,10 +29,12 @@ set(HIP_FILES csrc/ops.hip csrc/kernels.hip)
set(MPS_FILES csrc/mps_ops.mm)
set(METAL_FILES csrc/mps_kernels.metal)
set(XPU_FILES csrc/xpu_ops.cpp csrc/xpu_kernels.cpp)
set(CMAKE_CXX_COMPILER "nvcc")
# C++ sources are always included
list(APPEND SRC_FILES ${CPP_FILES})
set(COMPUTE_BACKEND "cpu" CACHE STRING "The compute backend to use (cpu, cuda, hip, mps, xpu)")
set(COMPUTE_BACKEND "cuda" CACHE STRING "The compute backend to use (cpu, cuda, hip, mps, xpu)")
#set(COMPUTE_BACKEND "cpu" CACHE STRING "The compute backend to use (cpu, cuda, hip, mps, xpu)")
set_property(CACHE COMPUTE_BACKEND PROPERTY STRINGS cpu cuda hip mps xpu)
option(PTXAS_VERBOSE "Pass through -v flag to PTX Assembler" OFF)
......@@ -188,7 +190,8 @@ if(BUILD_CUDA)
string(APPEND BNB_OUTPUT_NAME "_cuda${CUDA_VERSION_SHORT}")
add_compile_definitions(BUILD_CUDA)
elseif(BUILD_HIP)
enable_language(HIP)
#enable_language(HIP)
set(amd_comgr_DIR "/opt/dtk/lib64/cmake/amd_comgr")
message(STATUS "HIP Compiler: ${CMAKE_HIP_COMPILER}")
if(DEFINED BNB_ROCM_ARCH)
set(CMAKE_HIP_ARCHITECTURES ${BNB_ROCM_ARCH})
......@@ -287,8 +290,8 @@ if(BUILD_HIP)
find_package_and_print_version(hipsparse REQUIRED)
## hacky way of excluding hip::amdhip64 (with it linked many tests unexpectedly fail e.g. adam8bit because of inaccuracies)
set_target_properties(hip::host PROPERTIES INTERFACE_LINK_LIBRARIES "")
set_target_properties(hip-lang::host PROPERTIES INTERFACE_LINK_LIBRARIES "")
#set_target_properties(hip::host PROPERTIES INTERFACE_LINK_LIBRARIES "")
#set_target_properties(hip-lang::host PROPERTIES INTERFACE_LINK_LIBRARIES "")
set(CMAKE_HIP_IMPLICIT_LINK_LIBRARIES "")
target_include_directories(bitsandbytes PRIVATE ${CMAKE_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/include ${ROCM_PATH}/include /include)
......
README.md
View file @ 74f7aa06
<p align="center"><img src="https://avatars.githubusercontent.com/u/175231607?s=200&v=4" alt=""></p>
<h1 align="center">bitsandbytes</h1>
<p align="center">
<a href="https://github.com/bitsandbytes-foundation/bitsandbytes/main/LICENSE"><img alt="License" src="https://img.shields.io/github/license/bitsandbytes-foundation/bitsandbytes.svg?color=blue"></a>
<a href="https://pepy.tech/project/bitsandbytes"><img alt="Downloads" src="https://static.pepy.tech/badge/bitsandbytes/month"></a>
<a href="https://github.com/bitsandbytes-foundation/bitsandbytes/actions/workflows/tests.yml"><img alt="Nightly Unit Tests" src="https://img.shields.io/github/actions/workflow/status/bitsandbytes-foundation/bitsandbytes/tests.yml?logo=github&label=Nightly%20Tests"></a>
<a href="https://github.com/bitsandbytes-foundation/bitsandbytes/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/bitsandbytes-foundation/bitsandbytes"></a>
<a href="https://pypi.org/project/bitsandbytes/"><img alt="PyPI - Python Version" src="https://img.shields.io/pypi/pyversions/bitsandbytes"></a>
</p>
`bitsandbytes` enables accessible large language models via k-bit quantization for PyTorch. We provide three main features for dramatically reducing memory consumption for inference and training:
* 8-bit optimizers uses block-wise quantization to maintain 32-bit performance at a small fraction of the memory cost.
* LLM.int8() or 8-bit quantization enables large language model inference with only half the required memory and without any performance degradation. This method is based on vector-wise quantization to quantize most features to 8-bits and separately treating outliers with 16-bit matrix multiplication.
* QLoRA or 4-bit quantization enables large language model training with several memory-saving techniques that don't compromise performance. This method quantizes a model to 4-bits and inserts a small set of trainable low-rank adaptation (LoRA) weights to allow training.
The library includes quantization primitives for 8-bit & 4-bit operations, through `bitsandbytes.nn.Linear8bitLt` and `bitsandbytes.nn.Linear4bit` and 8-bit optimizers through `bitsandbytes.optim` module.
## System Requirements
bitsandbytes has the following minimum requirements for all platforms:
* Python 3.9+
* [PyTorch](https://pytorch.org/get-started/locally/) 2.3+
* _Note: While we aim to provide wide backwards compatibility, we recommend using the latest version of PyTorch for the best experience._
#### Accelerator support:
<small>Note: this table reflects the status of the current development branch. For the latest stable release, see the
[document in the 0.48.0 tag](https://github.com/bitsandbytes-foundation/bitsandbytes/blob/0.48.0/README.md#accelerator-support).
</small>
##### Legend:
🚧 = In Development,
〰️ = Partially Supported,
✅ = Supported,
❌ = Not Supported
<table>
<thead>
<tr>
<th>Platform</th>
<th>Accelerator</th>
<th>Hardware Requirements</th>
<th>LLM.int8()</th>
<th>QLoRA 4-bit</th>
<th>8-bit Optimizers</th>
</tr>
</thead>
<tbody>
<tr>
<td colspan="6">🐧 <strong>Linux, glibc >= 2.24</strong></td>
</tr>
<tr>
<td align="right">x86-64</td>
<td>◻️ CPU</td>
<td>AVX2</td>
<td>✅</td>
<td>✅</td>
<td>❌</td>
</tr>
<tr>
<td></td>
<td>🟩 NVIDIA GPU <br><code>cuda</code></td>
<td>SM60+ minimum<br>SM75+ recommended</td>
<td>✅</td>
<td>✅</td>
<td>✅</td>
</tr>
<tr>
<td></td>
<td>🟥 AMD GPU <br><code>cuda</code></td>
<td>
CDNA: gfx90a, gfx942<br>
RDNA: gfx1100
</td>
<td>🚧</td>
<td>🚧</td>
<td>🚧</td>
</tr>
<tr>
<td></td>
<td>🟦 Intel GPU <br><code>xpu</code></td>
<td>
Data Center GPU Max Series<br>
Arc A-Series (Alchemist)<br>
Arc B-Series (Battlemage)
</td>
<td>✅</td>
<td>✅</td>
<td>〰️</td>
</tr>
<tr>
<td></td>
<td>🟪 Intel Gaudi <br><code>hpu</code></td>
<td>Gaudi2, Gaudi3</td>
<td>✅</td>
<td>〰️</td>
<td>❌</td>
</tr>
<tr>
<td align="right">aarch64</td>
<td>◻️ CPU</td>
<td></td>
<td>✅</td>
<td>✅</td>
<td>❌</td>
</tr>
<tr>
<td></td>
<td>🟩 NVIDIA GPU <br><code>cuda</code></td>
<td>SM75+</td>
<td>✅</td>
<td>✅</td>
<td>✅</td>
</tr>
<tr>
<td colspan="6">🪟 <strong>Windows 11 / Windows Server 2019+</strong></td>
</tr>
<tr>
<td align="right">x86-64</td>
<td>◻️ CPU</td>
<td>AVX2</td>
<td>✅</td>
<td>✅</td>
<td>❌</td>
</tr>
<tr>
<td></td>
<td>🟩 NVIDIA GPU <br><code>cuda</code></td>
<td>SM60+ minimum<br>SM75+ recommended</td>
<td>✅</td>
<td>✅</td>
<td>✅</td>
</tr>
<tr>
<td></td>
<td>🟦 Intel GPU <br><code>xpu</code></td>
<td>
Arc A-Series (Alchemist) <br>
Arc B-Series (Battlemage)
</td>
<td>✅</td>
<td>✅</td>
<td>〰️</td>
</tr>
<tr>
<td colspan="6">🍎 <strong>macOS 14+</strong></td>
</tr>
<tr>
<td align="right">arm64</td>
<td>◻️ CPU</td>
<td>Apple M1+</td>
<td>🚧</td>
<td>🚧</td>
<td>❌</td>
</tr>
<tr>
<td></td>
<td>⬜ Metal <br><code>mps</code></td>
<td>Apple M1+</td>
<td>🚧</td>
<td>🚧</td>
<td>❌</td>
</tbody>
</table>
## :book: Documentation
* [Official Documentation](https://huggingface.co/docs/bitsandbytes/main)
* 🤗 [Transformers](https://huggingface.co/docs/transformers/quantization/bitsandbytes)
* 🤗 [Diffusers](https://huggingface.co/docs/diffusers/quantization/bitsandbytes)
* 🤗 [PEFT](https://huggingface.co/docs/peft/developer_guides/quantization#quantize-a-model)
# <div align="center"><strong>BitsandTytes</strong></div>
## 简介
bitsandbytes 通过为 PyTorch 提供 k 比特量化技术，使得大型语言模型的使用更加便捷。我们提供了三大核心功能，可显著降低推理和训练过程中的内存消耗：8 比特优化器：采用分块量化（block-wise quantization）技术，在仅占用少量内存的情况下，保持与 32 比特相近的性能。LLM.int8() 或 8 比特量化：实现大型语言模型的高效推理，所需内存仅为原来的一半，且不会造成任何性能下降。该方法基于逐向量量化（vector-wise quantization），将大部分特征量化为 8 比特，同时对异常值（outliers）单独使用 16 比特矩阵乘法进行处理。QLoRA 或 4 比特量化：通过多种节省内存的技术实现大型语言模型的训练，同时不牺牲性能。该方法将模型量化至 4 比特，并插入少量可训练的低秩适配（LoRA）权重，从而支持模型微调。核心功能在DCU加速卡的可用性，还针对DCU特有的硬件架构进行了深度定制优化，这使得开发者能够以极低的成本，轻松实现应用程序在DCU加速卡上的快速迁移和性能提升。目前已适配支持Pytorch2.4.1. Pytorch2.5.1. Pytorch2.7.1
## 安装
组件支持组合
| PyTorch版本 | fastpt版本 |bitsandbytes 版本 | DTK版本 | Python版本 | 推荐编译方式 |
| ----------- | ----------- | -------------------- | ------------ | ---------------- | ------------ |
| 2.5.1 | 2.1.0 |0.48.0 | >= 25.04 | 3.8、3.10、3.11 | fastpt不转码 |
| 2.4.1 | 2.0.1 |0.48.0 | >= 25.04 | 3.8、3.10、3.11 | fastpt不转码 |
| 其他 | 其他 | 其他 | 其他 | 3.8、3.10、3.11 | hip转码 |
+ pytorch版本大于2.4.1 && dtk版本大于25.04 推荐使用fastpt不转码编译。
- 安装相关依赖
```shell
pip install 'urllib3==1.26.14'
pip install pytest
pip install wheel
pip install -r requirements.txt
```
## :heart: Sponsors
The continued maintenance and development of `bitsandbytes` is made possible thanks to the generous support of our sponsors. Their contributions help ensure that we can keep improving the project and delivering valuable updates to the community.
### 使用pip方式安装
bitsandbytes whl包下载目录：[光合开发者社区](https://download.sourcefind.cn:65024/4/main/bitsandbytes)，选择对应的pytorch版本和python版本下载对应bitsandbytes的whl包
```shell
pip install torch* (下载torch的whl包)
pip install fastpt* --no-deps (下载fastpt的whl包)
source /usr/local/bin/fastpt -E
pip install bitsandbytes* (下载的bitsandbytes的whl包)
```
<kbd><a href="https://hf.co" target="_blank"><img width="100" src="https://huggingface.co/datasets/huggingface/brand-assets/resolve/main/hf-logo.svg" alt="Hugging Face"></a></kbd>
&nbsp;
<kbd><a href="https://intel.com" target="_blank"><img width="100" src="https://avatars.githubusercontent.com/u/17888862?s=100&v=4" alt="Intel"></a></kbd>
### 使用源码编译方式安装
## License
`bitsandbytes` is MIT licensed.
#### 编译环境准备
提供基于fastpt不转码编译：
We thank Fabio Cannizzo for his work on [FastBinarySearch](https://github.com/fabiocannizzo/FastBinarySearch) which we use for CPU quantization.
1. 基于光源pytorch基础镜像环境：镜像下载地址：[光合开发者社区](https://sourcefind.cn/#/image/dcu/pytorch)，根据pytorch、python、dtk及系统下载对应的镜像版本。
## How to cite us
If you found this library useful, please consider citing our work:
2. 基于现有python环境：安装pytorch，fastpt whl包下载目录：[光合开发者社区](https://sourcefind.cn/#/image/dcu/pytorch)，根据python、dtk版本,下载对应pytorch的whl包。安装命令如下：
```shell
pip install torch* (下载torch的whl包)
pip install fastpt* --no-deps (下载fastpt的whl包, 安装顺序，先安装torch，后安装fastpt)
```
#### 源码编译安装
```shell
git clone http://developer.sourcefind.cn/codes/OpenDAS/bitsandbytes.git
```
- 提供2种源码编译方式（进入bitsandbytes目录）：
1. 设置不转码编译环境变量
```shell
source /usr/local/bin/fastpt -C
```
### QLoRA
2. 编译whl包并安装
```shell
cd bitsandbytes
pip install -e .
python setup.py bdist_wheel
pip install dist/*.whl
```
```bibtex
@article{dettmers2023qlora,
title={Qlora: Efficient finetuning of quantized llms},
author={Dettmers, Tim and Pagnoni, Artidoro and Holtzman, Ari and Zettlemoyer, Luke},
journal={arXiv preprint arXiv:2305.14314},
year={2023}
}
3. 源码编译安装
```shell
python3 setup.py install
```
#### 注意事项
+ 若使用pip install下载安装过慢，可添加pypi清华源：-i https://pypi.tuna.tsinghua.edu.cn/simple/
+ ROCM_PATH为dtk的路径，默认为/opt/dtk
### LLM.int8()
## 验证
python -c "import bitsandbytes; bitsandbytes.\_\_version__"，版本号与官方版本同步，查询该软件的版本号，例如0.48.0.dev0
```bibtex
@article{dettmers2022llmint8,
title={LLM.int8(): 8-bit Matrix Multiplication for Transformers at Scale},
author={Dettmers, Tim and Lewis, Mike and Belkada, Younes and Zettlemoyer, Luke},
journal={arXiv preprint arXiv:2208.07339},
year={2022}
}
## 单测
```shell
cd bitsandbytes
pytest -vs ./tests
```
### 8-bit Optimizers
## Known Issue
无
```bibtex
@article{dettmers2022optimizers,
title={8-bit Optimizers via Block-wise Quantization},
author={Dettmers, Tim and Lewis, Mike and Shleifer, Sam and Zettlemoyer, Luke},
journal={9th International Conference on Learning Representations, ICLR},
year={2022}
}
```
## 参考资料
[README_ORIGIN.md](README_ORIGIN.md)
[https://github.com/bitsandbytes-foundation/bitsandbytes](https://github.com/bitsandbytes-foundation/bitsandbytes)
README_ORIGIN.md 0 → 100644
View file @ 74f7aa06
<p align="center"><img src="https://avatars.githubusercontent.com/u/175231607?s=200&v=4" alt=""></p>
<h1 align="center">bitsandbytes</h1>
<p align="center">
<a href="https://github.com/bitsandbytes-foundation/bitsandbytes/main/LICENSE"><img alt="License" src="https://img.shields.io/github/license/bitsandbytes-foundation/bitsandbytes.svg?color=blue"></a>
<a href="https://pepy.tech/project/bitsandbytes"><img alt="Downloads" src="https://static.pepy.tech/badge/bitsandbytes/month"></a>
<a href="https://github.com/bitsandbytes-foundation/bitsandbytes/actions/workflows/tests.yml"><img alt="Nightly Unit Tests" src="https://img.shields.io/github/actions/workflow/status/bitsandbytes-foundation/bitsandbytes/tests.yml?logo=github&label=Nightly%20Tests"></a>
<a href="https://github.com/bitsandbytes-foundation/bitsandbytes/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/bitsandbytes-foundation/bitsandbytes"></a>
<a href="https://pypi.org/project/bitsandbytes/"><img alt="PyPI - Python Version" src="https://img.shields.io/pypi/pyversions/bitsandbytes"></a>
</p>
`bitsandbytes` enables accessible large language models via k-bit quantization for PyTorch. We provide three main features for dramatically reducing memory consumption for inference and training:
* 8-bit optimizers uses block-wise quantization to maintain 32-bit performance at a small fraction of the memory cost.
* LLM.int8() or 8-bit quantization enables large language model inference with only half the required memory and without any performance degradation. This method is based on vector-wise quantization to quantize most features to 8-bits and separately treating outliers with 16-bit matrix multiplication.
* QLoRA or 4-bit quantization enables large language model training with several memory-saving techniques that don't compromise performance. This method quantizes a model to 4-bits and inserts a small set of trainable low-rank adaptation (LoRA) weights to allow training.
The library includes quantization primitives for 8-bit & 4-bit operations, through `bitsandbytes.nn.Linear8bitLt` and `bitsandbytes.nn.Linear4bit` and 8-bit optimizers through `bitsandbytes.optim` module.
## System Requirements
bitsandbytes has the following minimum requirements for all platforms:
* Python 3.9+
* [PyTorch](https://pytorch.org/get-started/locally/) 2.3+
* _Note: While we aim to provide wide backwards compatibility, we recommend using the latest version of PyTorch for the best experience._
#### Accelerator support:
<small>Note: this table reflects the status of the current development branch. For the latest stable release, see the
[document in the 0.48.0 tag](https://github.com/bitsandbytes-foundation/bitsandbytes/blob/0.48.0/README.md#accelerator-support).
</small>
##### Legend:
🚧 = In Development,
〰️ = Partially Supported,
✅ = Supported,
❌ = Not Supported
<table>
<thead>
<tr>
<th>Platform</th>
<th>Accelerator</th>
<th>Hardware Requirements</th>
<th>LLM.int8()</th>
<th>QLoRA 4-bit</th>
<th>8-bit Optimizers</th>
</tr>
</thead>
<tbody>
<tr>
<td colspan="6">🐧 <strong>Linux, glibc >= 2.24</strong></td>
</tr>
<tr>
<td align="right">x86-64</td>
<td>◻️ CPU</td>
<td>AVX2</td>
<td>✅</td>
<td>✅</td>
<td>❌</td>
</tr>
<tr>
<td></td>
<td>🟩 NVIDIA GPU <br><code>cuda</code></td>
<td>SM60+ minimum<br>SM75+ recommended</td>
<td>✅</td>
<td>✅</td>
<td>✅</td>
</tr>
<tr>
<td></td>
<td>🟥 AMD GPU <br><code>cuda</code></td>
<td>
CDNA: gfx90a, gfx942<br>
RDNA: gfx1100
</td>
<td>🚧</td>
<td>🚧</td>
<td>🚧</td>
</tr>
<tr>
<td></td>
<td>🟦 Intel GPU <br><code>xpu</code></td>
<td>
Data Center GPU Max Series<br>
Arc A-Series (Alchemist)<br>
Arc B-Series (Battlemage)
</td>
<td>✅</td>
<td>✅</td>
<td>〰️</td>
</tr>
<tr>
<td></td>
<td>🟪 Intel Gaudi <br><code>hpu</code></td>
<td>Gaudi2, Gaudi3</td>
<td>✅</td>
<td>〰️</td>
<td>❌</td>
</tr>
<tr>
<td align="right">aarch64</td>
<td>◻️ CPU</td>
<td></td>
<td>✅</td>
<td>✅</td>
<td>❌</td>
</tr>
<tr>
<td></td>
<td>🟩 NVIDIA GPU <br><code>cuda</code></td>
<td>SM75+</td>
<td>✅</td>
<td>✅</td>
<td>✅</td>
</tr>
<tr>
<td colspan="6">🪟 <strong>Windows 11 / Windows Server 2019+</strong></td>
</tr>
<tr>
<td align="right">x86-64</td>
<td>◻️ CPU</td>
<td>AVX2</td>
<td>✅</td>
<td>✅</td>
<td>❌</td>
</tr>
<tr>
<td></td>
<td>🟩 NVIDIA GPU <br><code>cuda</code></td>
<td>SM60+ minimum<br>SM75+ recommended</td>
<td>✅</td>
<td>✅</td>
<td>✅</td>
</tr>
<tr>
<td></td>
<td>🟦 Intel GPU <br><code>xpu</code></td>
<td>
Arc A-Series (Alchemist) <br>
Arc B-Series (Battlemage)
</td>
<td>✅</td>
<td>✅</td>
<td>〰️</td>
</tr>
<tr>
<td colspan="6">🍎 <strong>macOS 14+</strong></td>
</tr>
<tr>
<td align="right">arm64</td>
<td>◻️ CPU</td>
<td>Apple M1+</td>
<td>🚧</td>
<td>🚧</td>
<td>❌</td>
</tr>
<tr>
<td></td>
<td>⬜ Metal <br><code>mps</code></td>
<td>Apple M1+</td>
<td>🚧</td>
<td>🚧</td>
<td>❌</td>
</tbody>
</table>
## :book: Documentation
* [Official Documentation](https://huggingface.co/docs/bitsandbytes/main)
* 🤗 [Transformers](https://huggingface.co/docs/transformers/quantization/bitsandbytes)
* 🤗 [Diffusers](https://huggingface.co/docs/diffusers/quantization/bitsandbytes)
* 🤗 [PEFT](https://huggingface.co/docs/peft/developer_guides/quantization#quantize-a-model)
## :heart: Sponsors
The continued maintenance and development of `bitsandbytes` is made possible thanks to the generous support of our sponsors. Their contributions help ensure that we can keep improving the project and delivering valuable updates to the community.
<kbd><a href="https://hf.co" target="_blank"><img width="100" src="https://huggingface.co/datasets/huggingface/brand-assets/resolve/main/hf-logo.svg" alt="Hugging Face"></a></kbd>
&nbsp;
<kbd><a href="https://intel.com" target="_blank"><img width="100" src="https://avatars.githubusercontent.com/u/17888862?s=100&v=4" alt="Intel"></a></kbd>
## License
`bitsandbytes` is MIT licensed.
We thank Fabio Cannizzo for his work on [FastBinarySearch](https://github.com/fabiocannizzo/FastBinarySearch) which we use for CPU quantization.
## How to cite us
If you found this library useful, please consider citing our work:
### QLoRA
```bibtex
@article{dettmers2023qlora,
title={Qlora: Efficient finetuning of quantized llms},
author={Dettmers, Tim and Pagnoni, Artidoro and Holtzman, Ari and Zettlemoyer, Luke},
journal={arXiv preprint arXiv:2305.14314},
year={2023}
}
```
### LLM.int8()
```bibtex
@article{dettmers2022llmint8,
title={LLM.int8(): 8-bit Matrix Multiplication for Transformers at Scale},
author={Dettmers, Tim and Lewis, Mike and Belkada, Younes and Zettlemoyer, Luke},
journal={arXiv preprint arXiv:2208.07339},
year={2022}
}
```
### 8-bit Optimizers
```bibtex
@article{dettmers2022optimizers,
title={8-bit Optimizers via Block-wise Quantization},
author={Dettmers, Tim and Lewis, Mike and Shleifer, Sam and Zettlemoyer, Luke},
journal={9th International Conference on Learning Representations, ICLR},
year={2022}
}
```
bitsandbytes/backends/cuda/ops.py
View file @ 74f7aa06
......@@ -211,10 +211,7 @@ def _get_col_absmax(
def _(A: torch.Tensor, code: torch.Tensor, blocksize: int) -> tuple[torch.Tensor, torch.Tensor]:
torch._check_is_size(blocksize)
if HIP_ENVIRONMENT:
torch._check(blocksize in [4096, 2048, 1024, 512, 256, 128])
else:
torch._check(blocksize in [4096, 2048, 1024, 512, 256, 128, 64])
torch._check(code.dtype == torch.float32, lambda: f"code must be float32, got {code.dtype}")
......@@ -269,10 +266,8 @@ def _(
def _dequantize_blockwise_impl(
A: torch.Tensor, absmax: torch.Tensor, code: torch.Tensor, blocksize: int, dtype: torch.dtype, out: torch.Tensor
) -> None:
if HIP_ENVIRONMENT:
torch._check(blocksize in [4096, 2048, 1024, 512, 256, 128])
else:
torch._check(blocksize in [4096, 2048, 1024, 512, 256, 128, 64])
torch._check(A.dtype == torch.uint8, lambda: f"A must be uint8, got {A.dtype}")
torch._check(
......@@ -303,10 +298,8 @@ def _dequantize_blockwise_impl(
def _(
A: torch.Tensor, blocksize: int, quant_type: str, quant_storage: torch.dtype
) -> tuple[torch.Tensor, torch.Tensor]:
if HIP_ENVIRONMENT:
torch._check(blocksize in [4096, 2048, 1024, 512, 256, 128])
else:
torch._check(blocksize in [4096, 2048, 1024, 512, 256, 128, 64])
torch._check(quant_type in ["fp4", "nf4"])
torch._check(
......@@ -385,10 +378,8 @@ def _dequantize_4bit_impl(
dtype: torch.dtype,
out: torch.Tensor,
) -> None:
if HIP_ENVIRONMENT:
torch._check(blocksize in [4096, 2048, 1024, 512, 256, 128])
else:
torch._check(blocksize in [4096, 2048, 1024, 512, 256, 128, 64])
torch._check(quant_type in ["fp4", "nf4"])
torch._check(
......
bitsandbytes/functional.py
View file @ 74f7aa06
......@@ -806,7 +806,7 @@ def quantize_fp4(
quant_storage=torch.uint8,
):
if blocksize is None:
blocksize = 64 if not HIP_ENVIRONMENT else 128
blocksize = 128
return quantize_4bit(A, absmax, out, blocksize, compress_statistics, "fp4", quant_storage)
......@@ -819,7 +819,7 @@ def quantize_nf4(
quant_storage=torch.uint8,
):
if blocksize is None:
blocksize = 64 if not HIP_ENVIRONMENT else 128
blocksize = 128
return quantize_4bit(A, absmax, out, blocksize, compress_statistics, "nf4", quant_storage)
......@@ -857,7 +857,7 @@ def quantize_4bit(
"""
if blocksize is None:
blocksize = 64 if not HIP_ENVIRONMENT else 128
blocksize = 128
input_shape = A.shape
......@@ -912,7 +912,7 @@ def dequantize_fp4(
blocksize: Optional[int] = None,
) -> torch.Tensor:
if blocksize is None:
blocksize = 64 if not HIP_ENVIRONMENT else 128
blocksize = 128
return dequantize_4bit(A, quant_state, absmax, out, blocksize, "fp4")
......@@ -924,7 +924,7 @@ def dequantize_nf4(
blocksize: Optional[int] = None,
) -> torch.Tensor:
if blocksize is None:
blocksize = 64 if not HIP_ENVIRONMENT else 128
blocksize = 128
return dequantize_4bit(A, quant_state, absmax, out, blocksize, "nf4")
......@@ -964,7 +964,7 @@ def dequantize_4bit(
"""
if blocksize is None:
blocksize = 64 if not HIP_ENVIRONMENT else 128
blocksize = 128
if quant_state is None:
assert absmax is not None and out is not None
......
bitsandbytes/nn/modules.py
View file @ 74f7aa06
......@@ -221,7 +221,7 @@ class Params4bit(torch.nn.Parameter):
data = torch.empty(0)
if blocksize is None:
blocksize = 64 if not HIP_ENVIRONMENT else 128
blocksize = 128
self = torch.Tensor._make_subclass(cls, data, requires_grad)
self.blocksize = blocksize
......
csrc/common.cuh
View file @ 74f7aa06
......@@ -23,7 +23,7 @@
#define BNB_BF16_AVAILABLE (__CUDA_ARCH__ >= BNB_CC_AMPERE)
#define BNB_FP8_AVAILABLE (__CUDA_ARCH__ >= BNB_CC_ADA)
#define BNB_WARP_SIZE 32
#define BNB_WARP_SIZE warpSize
// The maximum number of resident threads per SM varies by arch.
// For A100/H100 and all prior to Turing, it is 2048, which allows
......
csrc/kernels.cu
View file @ 74f7aa06
......@@ -28,6 +28,7 @@
#define TH 1024
#define NUM 4
#define NUM_BLOCK 4096
#define DU_WARP_SIZE warpSize
__device__ static float fp4_dequantization_lut[8] = {
0.0f, // 0b000
......@@ -119,7 +120,62 @@ __device__ unsigned char dQuantizeFP4(float x) {
return 0b0000 + sign;
}
__device__ __forceinline__ float dDequantizeNF4(unsigned char val) { return nf4_dequantization_lut[val & 0x0F]; }
// __device__ __forceinline__ float dDequantizeNF4(unsigned char val) { return nf4_dequantization_lut[val & 0x0F]; }
__device__ __forceinline__ float dDequantizeNF4(unsigned char val)
{
// the values for this tree was generated by test_normal_map_tree
// in the file tests/test_functional.py
if((val & 0b1000) == 8)
if((val & 0b0100) == 4) // 1
if((val & 0b0010) == 2) // 11
if((val & 0b0001) == 1) // 111
return 1.0f;
else
return 0.7229568362236023f;
else
if((val & 0b0001) == 1) // 110
return 0.5626170039176941f;
else
return 0.44070982933044434f;
else
if((val & 0b0010) == 2) //10
if((val & 0b0001) == 1) // 101
return 0.33791524171829224f;
else
return 0.24611230194568634f;
else
if((val & 0b0001) == 1) // 100
return 0.16093020141124725f;
else
return 0.07958029955625534f;
else
if((val & 0b0100) == 4) // 0
if((val & 0b0010) == 2) //01
if((val & 0b0001) == 1) // 011
return 0.0f;
else
return -0.09105003625154495f;
else
if((val & 0b0001) == 1) // 010
return -0.18477343022823334f;
else
return -0.28444138169288635f;
else
if((val & 0b0010) == 2) //00
if((val & 0b0001) == 1) // 001
return -0.39491748809814453f;
else
return -0.5250730514526367f;
else
if((val & 0b0001) == 1) // 000
return -0.6961928009986877f;
else
return -1.0f;
}
__device__ unsigned char dQuantizeNF4(float x) {
......@@ -324,7 +380,7 @@ __launch_bounds__(TH, 4) __global__
template <typename T, int BLOCK_SIZE, int NUM_PER_TH, int STOCHASTIC, int DATA_TYPE>
//__launch_bounds__(TH, 4)
__global__ void kQuantizeBlockwise(
__global__ __launch_bounds__(1024) void kQuantizeBlockwise(
float* code, T* __restrict__ const A, float* absmax, unsigned char* out, float* __restrict__ const rand,
const int rand_offset, const int n
) {
......@@ -422,7 +478,7 @@ __global__ void kQuantizeBlockwise(
}
template <typename T, int TILE_SIZE, int THREADS, int NUM_PER_TH, int DATA_TYPE>
__global__ void
__global__ __launch_bounds__(1024) void
kDequantizeBlockwise(float* code, unsigned char* A, float* absmax, T* out, const int blocksize, const int n) {
const int n_load = (gridDim.x * TILE_SIZE);
......@@ -1877,7 +1933,7 @@ template __global__ void kInt8VectorQuant<half, 1024, 1>(
#define MM_DEQUANT_CONST 6.200012e-05f // 1.0f/(127.0f*127.0f)
template <int ITEMS_PER_THREAD, int THREADS>
__global__ void kdequant_mm_int32_fp16(
__global__ __launch_bounds__(1024) void kdequant_mm_int32_fp16(
int* __restrict__ const A, float* __restrict__ const rowStats, float* __restrict__ const colStats, half* out,
half* __restrict__ const bias, const int numRows, const int numCols, const int n
) {
......@@ -1956,9 +2012,9 @@ __global__ void kspmm_coo_very_sparse_naive(
const int offset = local_max_idx == 0 ? 0 : offset_rowidx[local_max_idx - 1];
const int local_row_idx = rowidx[offset];
const int warp_id = threadIdx.x / 32;
const int warp_idx = threadIdx.x % 32;
const int warp_offset = (warp_id * 32) * SPMM_ITEMS;
const int warp_id = threadIdx.x / DU_WARP_SIZE;
const int warp_idx = threadIdx.x % DU_WARP_SIZE;
const int warp_offset = (warp_id * DU_WARP_SIZE) * SPMM_ITEMS;
const int num_items = BITS == 8 ? 8 : 8;
int idx_col_B = warp_offset;
int local_idx_col_B_offset = 0;
......@@ -2080,12 +2136,12 @@ __global__ void gemm_device(int M, int N, int K, T* __restrict__ const A, T* B,
#if __CUDA_ARCH__ >= 750
using namespace nvcuda;
int col_offset = blockIdx.x * 32;
const int warp_id = threadIdx.x / 32;
const int half_warp_id = threadIdx.x / 16;
const int half_warp_lane = threadIdx.x % 16;
int col_offset = blockIdx.x * DU_WARP_SIZE;
const int warp_id = threadIdx.x / DU_WARP_SIZE;
const int half_warp_id = threadIdx.x / (DU_WARP_SIZE / 2);
const int half_warp_lane = threadIdx.x % (DU_WARP_SIZE / 2);
const int batch_size_warps = (WARPS - 1) * 2;
const int val_per_iter = blockDim.x - 32;
const int val_per_iter = blockDim.x - DU_WARP_SIZE;
T local_A[4];
T local_B[128];
......@@ -2248,7 +2304,7 @@ __global__ void gemm_device(int M, int N, int K, T* __restrict__ const A, T* B,
return;
}
// only warp_id == (WARPS-1) from here
int warp_lane = threadIdx.x % 32;
int warp_lane = threadIdx.x % DU_WARP_SIZE;
ticktock = ticktock == 0 ? 1 : 0;
for (int k = 0; k < batch_size_warps; k++) {
......@@ -2298,11 +2354,11 @@ __global__ void kgemm_4bit_inference(
//// 9. write outputs to matmul output matrix
#if __CUDA_ARCH__ >= 750
using namespace nvcuda;
int col_offset = blockIdx.x * 32;
const int warp_id = threadIdx.x / 32;
const int warp_idx = threadIdx.x % 32;
const int half_warp_id = threadIdx.x / 16;
const int half_warp_lane = threadIdx.x % 16;
int col_offset = blockIdx.x * DU_WARP_SIZE;
const int warp_id = threadIdx.x / DU_WARP_SIZE;
const int warp_idx = threadIdx.x % DU_WARP_SIZE;
const int half_warp_id = threadIdx.x / (DU_WARP_SIZE / 2);
const int half_warp_lane = threadIdx.x % (DU_WARP_SIZE / 2);
const int batch_size_warps = (WARPS - 1) * 2;
T quant_map[16];
......@@ -2474,7 +2530,7 @@ __global__ void kgemm_4bit_inference(
return;
}
// only warp_id == (WARPS-1) from here
int warp_lane = threadIdx.x % 32;
int warp_lane = threadIdx.x % DU_WARP_SIZE;
ticktock = ticktock == 0 ? 1 : 0;
for (int k = 0; k < batch_size_warps; k++) {
......@@ -2509,11 +2565,11 @@ __global__ void kgemm_4bit_inference_naive(
// 4 warps -> 4 loads per iter
// 1x32 * 32x4 -> 1x4 outputs per thread block
typedef cub::WarpReduce<float> WarpReduce;
__shared__ typename WarpReduce::TempStorage temp_storage[THREADS / 32];
__shared__ typename WarpReduce::TempStorage temp_storage[THREADS / DU_WARP_SIZE];
const int warp_idx = threadIdx.x / 32;
const int warp_lane = threadIdx.x % 32;
const int row_B = (THREADS / 32) * blockIdx.x + warp_idx;
const int warp_idx = threadIdx.x / DU_WARP_SIZE;
const int warp_lane = threadIdx.x % DU_WARP_SIZE;
const int row_B = (THREADS / DU_WARP_SIZE) * blockIdx.x + warp_idx;
const int offset_B = ldb * row_B;
const int num_values_8bit = num_values_4bit / 2;
float local_C = 0.0f;
......@@ -2532,7 +2588,7 @@ __global__ void kgemm_4bit_inference_naive(
// A: [1, K]
// B: [N, K]
for (int inner_idx = warp_lane * num_values_4bit; inner_idx < K; inner_idx += 32 * num_values_4bit) {
for (int inner_idx = warp_lane * num_values_4bit; inner_idx < K; inner_idx += warpSize * num_values_4bit) {
const int inner_idx_halved = inner_idx / 2;
// Since blocksize will always be a power-of-2, we avoid more expensive
......@@ -2921,21 +2977,21 @@ MAKE_kQuantizeBlockwise(half, 1024, 4, 0, General8bit)
MAKE_kQuantizeBlockwise(half, 512, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(half, 256, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(half, 128, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(half, 64, 2, 0, General8bit)
// MAKE_kQuantizeBlockwise(half, 64, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(half, 4096, 4, 0, FP4)
MAKE_kQuantizeBlockwise(half, 2048, 4, 0, FP4)
MAKE_kQuantizeBlockwise(half, 1024, 4, 0, FP4)
MAKE_kQuantizeBlockwise(half, 512, 2, 0, FP4)
MAKE_kQuantizeBlockwise(half, 256, 2, 0, FP4)
MAKE_kQuantizeBlockwise(half, 128, 2, 0, FP4)
MAKE_kQuantizeBlockwise(half, 64, 2, 0, FP4)
// MAKE_kQuantizeBlockwise(half, 64, 2, 0, FP4)
MAKE_kQuantizeBlockwise(half, 4096, 4, 0, NF4)
MAKE_kQuantizeBlockwise(half, 2048, 4, 0, NF4)
MAKE_kQuantizeBlockwise(half, 1024, 4, 0, NF4)
MAKE_kQuantizeBlockwise(half, 512, 2, 0, NF4)
MAKE_kQuantizeBlockwise(half, 256, 2, 0, NF4)
MAKE_kQuantizeBlockwise(half, 128, 2, 0, NF4)
MAKE_kQuantizeBlockwise(half, 64, 2, 0, NF4)
// MAKE_kQuantizeBlockwise(half, 64, 2, 0, NF4)
MAKE_kQuantizeBlockwise(float, 4096, 4, 0, General8bit)
MAKE_kQuantizeBlockwise(float, 4096, 4, 1, General8bit)
MAKE_kQuantizeBlockwise(float, 2048, 4, 0, General8bit)
......@@ -2943,21 +2999,21 @@ MAKE_kQuantizeBlockwise(float, 1024, 4, 0, General8bit)
MAKE_kQuantizeBlockwise(float, 512, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(float, 256, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(float, 128, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(float, 64, 2, 0, General8bit)
// MAKE_kQuantizeBlockwise(float, 64, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(float, 4096, 4, 0, FP4)
MAKE_kQuantizeBlockwise(float, 2048, 4, 0, FP4)
MAKE_kQuantizeBlockwise(float, 1024, 4, 0, FP4)
MAKE_kQuantizeBlockwise(float, 512, 2, 0, FP4)
MAKE_kQuantizeBlockwise(float, 256, 2, 0, FP4)
MAKE_kQuantizeBlockwise(float, 128, 2, 0, FP4)
MAKE_kQuantizeBlockwise(float, 64, 2, 0, FP4)
// MAKE_kQuantizeBlockwise(float, 64, 2, 0, FP4)
MAKE_kQuantizeBlockwise(float, 4096, 4, 0, NF4)
MAKE_kQuantizeBlockwise(float, 2048, 4, 0, NF4)
MAKE_kQuantizeBlockwise(float, 1024, 4, 0, NF4)
MAKE_kQuantizeBlockwise(float, 512, 2, 0, NF4)
MAKE_kQuantizeBlockwise(float, 256, 2, 0, NF4)
MAKE_kQuantizeBlockwise(float, 128, 2, 0, NF4)
MAKE_kQuantizeBlockwise(float, 64, 2, 0, NF4)
// MAKE_kQuantizeBlockwise(float, 64, 2, 0, NF4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 4096, 4, 0, General8bit)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 4096, 4, 1, General8bit)
......@@ -2966,21 +3022,21 @@ MAKE_kQuantizeBlockwise(__nv_bfloat16, 1024, 4, 0, General8bit)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 512, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 256, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 128, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 64, 2, 0, General8bit)
// MAKE_kQuantizeBlockwise(__nv_bfloat16, 64, 2, 0, General8bit)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 4096, 4, 0, FP4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 2048, 4, 0, FP4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 1024, 4, 0, FP4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 512, 2, 0, FP4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 256, 2, 0, FP4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 128, 2, 0, FP4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 64, 2, 0, FP4)
// MAKE_kQuantizeBlockwise(__nv_bfloat16, 64, 2, 0, FP4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 4096, 4, 0, NF4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 2048, 4, 0, NF4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 1024, 4, 0, NF4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 512, 2, 0, NF4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 256, 2, 0, NF4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 128, 2, 0, NF4)
MAKE_kQuantizeBlockwise(__nv_bfloat16, 64, 2, 0, NF4)
// MAKE_kQuantizeBlockwise(__nv_bfloat16, 64, 2, 0, NF4)
template __global__ void kDequantizeBlockwise<half, 512, 64, 8, FP4>(
float* code, unsigned char* A, float* absmax, half* out, const int blocksize, const int n
......
csrc/ops.cu
View file @ 74f7aa06
......@@ -51,8 +51,8 @@ void quantizeBlockwise(
kQuantizeBlockwise<T, 256, 2, 0, DATA_TYPE><<<num_blocks, 128>>>(code, A, absmax, out, rand, rand_offset, n);
else if (blocksize == 128)
kQuantizeBlockwise<T, 128, 2, 0, DATA_TYPE><<<num_blocks, 64>>>(code, A, absmax, out, rand, rand_offset, n);
else if (blocksize == 64)
kQuantizeBlockwise<T, 64, 2, 0, DATA_TYPE><<<num_blocks, 32>>>(code, A, absmax, out, rand, rand_offset, n);
// else if (blocksize == 64)
// kQuantizeBlockwise<T, 64, 2, 0, DATA_TYPE><<<num_blocks, 32>>>(code, A, absmax, out, rand, rand_offset, n);
CUDA_CHECK_RETURN(cudaPeekAtLastError());
}
......@@ -381,11 +381,38 @@ int igemmlt(
checkCublasStatus(cublasLtMatmulDescSetAttribute(matmulDesc, CUBLASLT_MATMUL_DESC_TRANSA, &opT, sizeof(opT)));
if (DTYPE_OUT == 32) {
cublasLtMatmulPreference_t pref;
checkCublasStatus(cublasLtMatmulPreferenceCreate(&pref));
checkCublasStatus(
cublasLtMatmulPreferenceSetAttribute(pref,
CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES,
&max_workspace_size,
sizeof(max_workspace_size)));
const int request_solutions = 1;
cublasLtMatmulHeuristicResult_t heuristicResult[request_solutions];
int returnedAlgoCount = 0;
checkCublasStatus(cublasLtMatmulAlgoGetHeuristic(ltHandle,
matmulDesc,
aDesc,
bDesc,
cDesc,
cDesc,
pref,
request_solutions,
heuristicResult,
&returnedAlgoCount));
if (returnedAlgoCount == 0) {
has_error = 1;
fprintf(stderr, "Error: Matmul Algo Heuristic didn't return algorithms\n");
} else {
int alpha = 1, beta = 0;
has_error |= checkCublasStatus(cublasLtMatmul(
ltHandle, matmulDesc, &alpha, A, aDesc, B, bDesc, &beta, (int32_t*)C, cDesc, (int32_t*)C, cDesc, NULL, NULL,
ltHandle, matmulDesc, &alpha, A, aDesc, B, bDesc, &beta, (int32_t*)C, cDesc, (int32_t*)C, cDesc, &heuristicResult[0].algo, NULL,
0, stream
));
}
} else {
// This path is unlikely to be used, as 8-bit accumulation can lead to likely overflows.
......@@ -543,6 +570,10 @@ void gemm_4bit_inference_naive(
) {
int num_blocks = (m + 3) / 4;
if (64 == warpSize) {
num_blocks = (m + 1) / 2;
}
kgemm_4bit_inference_naive<T, 128, BITS>
<<<num_blocks, 128, 0, stream>>>(m, n, k, A, B, absmax, datatype, out, lda, ldb, ldc, blocksize);
CUDA_CHECK_RETURN(cudaPeekAtLastError());
......
tests/test_functional.py
View file @ 74f7aa06
......@@ -96,7 +96,7 @@ class Test8BitBlockwiseQuantizeFunctional:
@pytest.mark.parametrize("nested", TRUE_FALSE, ids=id_formatter("nested"))
@pytest.mark.parametrize(
"blocksize",
[4096, 2048, 1024, 512, 256, 128, 64] if not HIP_ENVIRONMENT else [4096, 2048, 1024, 512, 256, 128],
[4096, 2048, 1024, 512, 256, 128] if not HIP_ENVIRONMENT else [4096, 2048, 1024, 512, 256, 128],
)
@pytest.mark.parametrize("signed", TRUE_FALSE, ids=id_formatter("signed"))
def test_dynamic_blockwise_quantization(self, device, dtype, nested, blocksize, signed):
......@@ -836,6 +836,7 @@ class TestSpMMFunctional:
@pytest.mark.parametrize("dim2", [128, 512], ids=id_formatter("dim2"))
@pytest.mark.parametrize("transposed_B", TRUE_FALSE, ids=id_formatter("transposed_B"))
def test_spmm_coo(self, dim1, dim2, transposed_B):
pytest.skip("this test is not supported on ROCm yet")
threshold = 1.5
dim3 = torch.randint(32, 128, size=(1,)).item()
# dim3 = 17
......@@ -968,6 +969,7 @@ class TestSpMMFunctional:
@pytest.mark.parametrize("dim2", [2048])
@pytest.mark.parametrize("dtype", [torch.int8])
def test_spmm_coo_dequant(self, dim1, dim2, dtype):
pytest.skip("this test is not supported on ROCm yet")
threshold = 6.0
# threshold = 2.8
# threshold = 0.0
......@@ -1109,7 +1111,7 @@ class TestQuantize4BitFunctional:
@pytest.mark.parametrize("quant_type", ["fp4", "nf4"])
@pytest.mark.parametrize(
"blocksize",
[64, 128, 256, 512, 1024, 2048, 4096] if not HIP_ENVIRONMENT else [128, 256, 512, 1024, 2048, 4096],
[128, 256, 512, 1024, 2048, 4096] if not HIP_ENVIRONMENT else [128, 256, 512, 1024, 2048, 4096],
)
def test_4bit_quant(self, device, dtype, quant_type, blocksize):
if device == "hpu" and not is_supported_on_hpu(quant_type, dtype):
......@@ -1180,7 +1182,7 @@ class TestQuantize4BitFunctional:
@pytest.mark.parametrize("device", get_available_devices())
@pytest.mark.parametrize("quant_type", ["fp4", "nf4"])
@pytest.mark.parametrize("blocksize", [64, 128] if not HIP_ENVIRONMENT else [128], ids=id_formatter("blocksize"))
@pytest.mark.parametrize("blocksize", [128] if not HIP_ENVIRONMENT else [128], ids=id_formatter("blocksize"))
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16], ids=describe_dtype)
def test_4bit_compressed_stats(self, device, quant_type, blocksize, dtype):
if device == "hpu" and not is_supported_on_hpu(quant_type, dtype):
......
tests/test_ops.py
View file @ 74f7aa06
......@@ -102,7 +102,7 @@ class TestLLMInt8Ops:
class TestInt8BlockwiseQuantOps:
@pytest.mark.parametrize("device", get_available_devices())
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32], ids=id_formatter("dtype"))
@pytest.mark.parametrize("blocksize", [64, 128, 256, 512] if not HIP_ENVIRONMENT else [128, 256, 512])
@pytest.mark.parametrize("blocksize", [128, 256, 512] if not HIP_ENVIRONMENT else [128, 256, 512])
def test_quantize_blockwise(self, device, dtype, blocksize):
if device == "cpu":
if dtype != torch.float32:
......@@ -126,7 +126,7 @@ class TestInt8BlockwiseQuantOps:
@pytest.mark.parametrize("device", get_available_devices())
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32], ids=id_formatter("dtype"))
@pytest.mark.parametrize("blocksize", [64, 128, 256, 512] if not HIP_ENVIRONMENT else [128, 256, 512])
@pytest.mark.parametrize("blocksize", [128, 256, 512] if not HIP_ENVIRONMENT else [128, 256, 512])
def test_dequantize_blockwise(self, device, dtype, blocksize):
if device == "cpu" and dtype != torch.float32:
pytest.skip("CPU implementation is only available for float32")
......@@ -152,7 +152,7 @@ class Test4bitBlockwiseQuantOps:
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32], ids=id_formatter("dtype"))
@pytest.mark.parametrize("storage_dtype", [torch.uint8, torch.bfloat16], ids=id_formatter("storage_dtype"))
@pytest.mark.parametrize("quant_type", ["fp4", "nf4"])
@pytest.mark.parametrize("blocksize", [64, 128, 256, 512] if not HIP_ENVIRONMENT else [128, 256, 512])
@pytest.mark.parametrize("blocksize", [128, 256, 512] if not HIP_ENVIRONMENT else [128, 256, 512])
def test_quantize_4bit(self, device, dtype, storage_dtype, quant_type, blocksize):
if device == "hpu" and not is_supported_on_hpu(quant_type, dtype, storage_dtype):
pytest.skip("This configuration is not supported on HPU.")
......@@ -176,7 +176,7 @@ class Test4bitBlockwiseQuantOps:
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32], ids=id_formatter("dtype"))
@pytest.mark.parametrize("storage_dtype", [torch.uint8, torch.bfloat16], ids=id_formatter("storage_dtype"))
@pytest.mark.parametrize("quant_type", ["fp4", "nf4"])
@pytest.mark.parametrize("blocksize", [64, 128, 256, 512] if not HIP_ENVIRONMENT else [128, 256, 512])
@pytest.mark.parametrize("blocksize", [128, 256, 512] if not HIP_ENVIRONMENT else [128, 256, 512])
def test_dequantize_4bit(self, device, dtype, storage_dtype, quant_type, blocksize):
if device == "hpu" and not is_supported_on_hpu(quant_type, dtype, storage_dtype):
pytest.skip("This configuration is not supported on HPU.")
......@@ -210,7 +210,7 @@ class Test4bitBlockwiseQuantOps:
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32], ids=id_formatter("dtype"))
@pytest.mark.parametrize("storage_dtype", [torch.uint8, torch.bfloat16], ids=id_formatter("storage_dtype"))
@pytest.mark.parametrize("quant_type", ["fp4", "nf4"])
@pytest.mark.parametrize("blocksize", [64, 128, 256, 512] if not HIP_ENVIRONMENT else [128, 256, 512])
@pytest.mark.parametrize("blocksize", [128, 256, 512] if not HIP_ENVIRONMENT else [128, 256, 512])
@pytest.mark.skipif(HIP_ENVIRONMENT, reason="this test is not supported on ROCm yet")
def test_gemv_4bit(self, device, dtype, storage_dtype, quant_type, blocksize):
if device == "hpu" and not is_supported_on_hpu(quant_type, dtype, storage_dtype):
......
tests/test_parametrize.py
View file @ 74f7aa06
......@@ -39,7 +39,7 @@ class ParametrizeTestModule(nn.Module):
@pytest.mark.parametrize("compress_statistics", TRUE_FALSE, ids=id_formatter("compress_statistics"))
@pytest.mark.parametrize(
"blocksize",
[64, 128, 256] if not HIP_ENVIRONMENT else [128, 256],
[128, 256] if not HIP_ENVIRONMENT else [128, 256],
)
def test_replace_parameter_4bit(device, dtype, quant_type, compress_statistics, blocksize):
"""Test basic parameter replacement with 4-bit quantization on different dtypes."""
......@@ -267,7 +267,7 @@ def test_quant_state_preservation(device, dtype):
module = ParametrizeTestModule(device=device, dtype=dtype)
blocksize = 128 if HIP_ENVIRONMENT else 64
blocksize = 128
# Apply parametrization with specific settings
replace_parameter_4bit(module, "weight_2d", quant_type="nf4", compress_statistics=True, blocksize=blocksize)
......@@ -326,7 +326,7 @@ def test_multiple_parameters(device, dtype):
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16], ids=describe_dtype)
@pytest.mark.parametrize(
"blocksize",
[64, 128, 256] if not HIP_ENVIRONMENT else [128, 256],
[128, 256] if not HIP_ENVIRONMENT else [128, 256],
)
def test_different_blocksizes(device, dtype, blocksize):
"""Test parametrization with different block sizes to verify flexibility."""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment