v1.0

quantize/quantize_data/wikitext/wikitext/wikitext-2-raw-v1/0.0.0/b08601e04326c79dfdd32d625aee71d232d685c3/dataset_info.json

+{"description": "", "citation": "", "homepage": "", "license": "", "features": {"text": {"dtype": "string", "_type": "Value"}}, "builder_name": "parquet", "dataset_name": "wikitext", "config_name": "wikitext-2-raw-v1", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"test": {"name": "test", "num_bytes": 1305088, "num_examples": 4358, "dataset_name": "wikitext"}, "train": {"name": "train", "num_bytes": 11061717, "num_examples": 36718, "dataset_name": "wikitext"}, "validation": {"name": "validation", "num_bytes": 1159288, "num_examples": 3760, "dataset_name": "wikitext"}}, "download_checksums": {"hf://datasets/wikitext@b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/test-00000-of-00001.parquet": {"num_bytes": 732610, "checksum": null}, "hf://datasets/wikitext@b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/train-00000-of-00001.parquet": {"num_bytes": 6357543, "checksum": null}, "hf://datasets/wikitext@b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1/validation-00000-of-00001.parquet": {"num_bytes": 657209, "checksum": null}}, "download_size": 7747362, "dataset_size": 13526093, "size_in_bytes": 21273455}
\ No newline at end of file

quantize/quantize_eval.py

+import torch
+import torch.nn as nn
+from tqdm import tqdm
+from datasets import load_dataset
+from transformers import AutoModelForCausalLM, AutoTokenizer,AutoConfig
+import GPUtil
+import argparse
+
+parser = argparse.ArgumentParser(description="========量化困惑度测试========")
+parser.add_argument(
+    "--model_path",  
+    type=str,  
+    default='',  
+    help="未量化前的模型路径。"  
+)
+parser.add_argument(
+    "--bnb_path",  
+    type=str,  
+    default='',  
+    help="bnb量化后的模型路径。"  
+)
+parser.add_argument(
+    "--awq_path",  
+    type=str,  
+    default='',  
+    help="awq量化后的模型保存路径。"  
+)
+#we will support gptq later
+parser.add_argument(
+    "--gptq_path",  
+    type=str,  
+    default='',  
+    help="gptq量化后的模型保存路径。"  
+)
+parser.add_argument(
+    "--data_path",  
+    type=str,  
+    default='quantize_data/wikitext',  
+    help="可以是以后的量化数据集，示例中默认为wiki_text"  
+)
+
+def get_device():
+    if torch.backends.mps.is_available():
+        return "mps"
+    elif torch.cuda.is_available():
+        return "cuda:0"
+    else:
+        return "cpu"
+
+def evaluate_perplexity(model, tokenizer,data_path):
+    def _perplexity(nlls, n_samples, seqlen):
+        return torch.exp(torch.stack(nlls).sum() / (n_samples * seqlen))
+
+    data = load_dataset(data_path,  split="test")
+    data = tokenizer("\n\n".join(data["text"]), return_tensors="pt")
+    data = data.input_ids.to('cuda:0')
+
+    seqlen = 2048
+    model = model.eval()
+    n_samples = data.numel() // seqlen
+
+    nlls = []
+
+    with tqdm(range(n_samples), desc="Perplexity -") as progress_bar:
+        for i in progress_bar:
+            start_index = i * seqlen
+            end_index = (i + 1) * seqlen
+            batch = data[:, start_index:end_index].to('cuda:0')
+            with torch.no_grad():
+                logits = model(batch).logits
+            shift_logits = logits[:, :-1, :].contiguous().float()
+            shift_labels = data[:, start_index:end_index][:, 1:]
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)
+            )
+            neg_log_likelihood = loss.float() * seqlen
+            nlls.append(neg_log_likelihood)
+
+            curr_ppl = _perplexity(nlls, i + 1, seqlen)
+            progress_bar.set_description(f"Perplexity {curr_ppl:.3f}")
+
+    ppl = _perplexity(nlls, n_samples, seqlen)
+
+    return ppl.item()
+
+if __name__ == "__main__":
+    
+    args = parser.parse_args()
+
+    if args.model_path != "":
+        print("pretrained model：",args.model_path.split('/')[-1])
+        model = AutoModelForCausalLM.from_pretrained(args.model_path, torch_dtype=torch.bfloat16, device_map='cuda', trust_remote_code=True)
+        tokenizer = AutoTokenizer.from_pretrained(args.model_path)
+        gpu_usage = GPUtil.getGPUs()[0].memoryUsed 
+        print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
+        evaluate_perplexity(model, tokenizer, args.data_path)
+        del model
+
+    if args.awq_path != "":
+        from awq import AutoAWQForCausalLM
+        print("awq model：",args.awq_path.split('/')[-1])
+        model = AutoAWQForCausalLM.from_quantized(args.awq_path, fuse_layers=True,device_map={"":'cuda:0'})
+        tokenizer = AutoTokenizer.from_pretrained(args.awq_path)
+        gpu_usage = GPUtil.getGPUs()[0].memoryUsed 
+        print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
+        evaluate_perplexity(model, tokenizer, args.data_path)
+        del model
+
+#we will support the autogptq  later
+    if args.gptq_path != "":
+        from auto_gptq import AutoGPTQForCausalLM
+        print("gptq model：",args.gptq_path.split('/')[-1])
+        tokenizer = AutoTokenizer.from_pretrained(args.gptq_path, use_fast=True)
+        model = AutoGPTQForCausalLM.from_quantized(args.gptq_path, device="cuda:0",trust_remote_code=True)
+        gpu_usage = GPUtil.getGPUs()[0].memoryUsed 
+        print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
+        evaluate_perplexity(model, tokenizer, args.data_path)
+        del model
+    
+    if args.bnb_path != "":
+        from accelerate.utils import BnbQuantizationConfig
+        bnb_quantization_config = BnbQuantizationConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4")
+        print("bnb model：",args.gptq_path.split('/')[-1])
+        # config=AutoConfig.from_pretrained(args.bnb_path,trust_remote_code=True)
+        # bnb_config=config.quantization_config
+        tokenizer = AutoTokenizer.from_pretrained(args.bnb_path, use_fast=True)
+        model = AutoModelForCausalLM.from_pretrained(args.bnb_path, trust_remote_code=True,)#quantization_config=bnb_config,)
+        gpu_usage = GPUtil.getGPUs()[0].memoryUsed 
+        print(f"gpu usage: {round(gpu_usage/1024,2)}GB")
+        evaluate_perplexity(model, tokenizer, args.data_path)
+        del model

quantize/quantize_eval.sh

+#!/bin/bash
+
+awq_path="/root/ld/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit"
+gptq_path="/root/ld/pull_request/MiniCPM/quantize/gptq_cpm_1b_4bit"
+model_path="/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16"
+bnb_path="/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16_int4"
+python quantize_eval.py --awq_path "${awq_path}" \
+ --model_path "${model_path}" --gptq_path "${gptq_path}" --bnb_path "${bnb_path}"
\ No newline at end of file

quantize/readme.md

+## 模型量化
+<p id="gptq"></p>
+
+**gptq量化**
+1. 首先git获取[minicpm_gptqd代码](https://github.com/LDLINGLINGLING/AutoGPTQ/tree/minicpm_gptq)
+2. 进入minicpm_gptqd主目录./AutoGPTQ，命令行输入：
+    ```
+    pip install e .
+    ```
+3. 前往[模型下载](#1)下载未量化的MiniCPM仓库下所有文件放至本地同一文件夹下,1b、2b模型均可,训练后模型亦可。
+4. 命令行输入以下命令，其中no_quantized_model_path是第3步模型下载路径，save_path是量化模型保存路径，--bits 为量化位数可以选择输入4或者8
+    ```
+    cd Minicpm/quantize
+    python gptq_quantize.py --pretrained_model_dir no_quant_model_path --quantized_model_dir quant_save_path --bits 4
+    ```
+5. 可以使用./AutoGPTQ/examples/quantization/inference.py进行推理，也可以参考前文使用vllm对量化后的模型，单卡4090下minicpm-1b-int4模型vllm推理在2000token/s左右。
+
+<p id="awq"></p>
+
+**awq量化**
+1. 在quantize/awq_quantize.py 文件中修改根据注释修改配置参数：
+  ```python
+  model_path = '/root/ld/ld_model_pretrained/MiniCPM-1B-sft-bf16' # model_path or model_id
+  quant_path = '/root/ld/ld_project/pull_request/MiniCPM/quantize/awq_cpm_1b_4bit' # quant_save_path
+  quant_data_path='/root/ld/ld_project/pull_request/MiniCPM/quantize/quantize_data/wikitext'# 写入自带量化数据集，data下的alpaca或者wikitext
+  quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" } # "w_bit":4 or 8
+  quant_samples=512 # how many samples to use for calibration
+  custom_data=[{'question':'你叫什么名字。','answer':'我是openmbmb开源的小钢炮minicpm。'}, # 自定义数据集可用
+                 {'question':'你有什么特色。','answer':'我很小，但是我很强。'}]
+  ```
+2. 在quantize/quantize_data文件下已经提供了alpaca和wiki_text两个数据集作为量化校准集，修改上述quant_data_path为其中一个文件夹的路径
+3. 如果需要自定义数据集，修改quantize/awq_quantize.py中的custom_data变量，如：
+    ```python
+    custom_data=[{'question':'过敏性鼻炎有什么症状？','answer':'过敏性鼻炎可能鼻塞，流鼻涕，头痛等症状反复发作，严重时建议及时就医。'},
+                 {'question':'1+1等于多少？','answer':'等于2'}]
+    ```
+4. 根据选择的数据集，选择以下某一行代码替换 quantize/awq_quantize.py 中第三十八行：
+  ```python
+    #使用wikitext进行量化
+    model.quantize(tokenizer, quant_config=quant_config, calib_data=load_wikitext(quant_data_path=quant_data_path))
+    #使用alpaca进行量化
+    model.quantize(tokenizer, quant_config=quant_config, calib_data=load_alpaca(quant_data_path=quant_data_path))
+    #使用自定义数据集进行量化
+    model.quantize(tokenizer, quant_config=quant_config, calib_data=load_cust_data(quant_data_path=quant_data_path))
+    
+  ```
+5. 运行quantize/awq_quantize.py文件,在设置的quan_path目录下可得awq量化后的模型。
+
+<p id="bnb"></p>
+
+**bnb量化**
+1. 在quantize/bnb_quantize.py 文件中修改根据注释修改配置参数：
+```python
+model_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16"  # 模型地址
+save_path = "/root/ld/ld_model_pretrain/MiniCPM-1B-sft-bf16_int4"  # 量化模型保存地址
+```
+2. 更多量化参数可根据注释以及llm.int8()算法进行修改：
+```python
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,  # 是否进行4bit量化
+    load_in_8bit=False,  # 是否进行8bit量化
+    bnb_4bit_compute_dtype=torch.float16,  # 计算精度设置
+    bnb_4bit_quant_storage=torch.uint8,  # 量化权重的储存格式
+    bnb_4bit_quant_type="nf4",  # 量化格式，这里用的是正太分布的int4
+    bnb_4bit_use_double_quant=True,  # 是否采用双量化，即对zeropoint和scaling参数进行量化
+    llm_int8_enable_fp32_cpu_offload=False,  # 是否llm使用int8，cpu上保存的参数使用fp32
+    llm_int8_has_fp16_weight=False,  # 是否启用混合精度
+    #llm_int8_skip_modules=["out_proj", "kv_proj", "lm_head"],  # 不进行量化的模块
+    llm_int8_threshold=6.0,  # llm.int8()算法中的离群值，根据这个值区分是否进行量化
+)
+```
+3. 运行quantize/bnb_quantize.py文件,在设置的save_path目录下可得bnb量化后的模型。
+```python
+cd MiniCPM/quantize
+python bnb_quantize.py
+```
+<p id="quantize_test"></p>
+
+**量化测试**
+1. 命令行进入到 MiniCPM/quantize 目录下
+2. 修改quantize_eval.sh文件中awq_path,gptq_path,awq_path,bnb_path,如果不需要测试的类型保持为空字符串，如下示例表示仅测试awq模型：
+  ```
+    awq_path="/root/ld/ld_project/AutoAWQ/examples/awq_cpm_1b_4bit"
+    gptq_path=""
+    model_path=""
+    bnb_path=""
+  ```
+3. 在MiniCPM/quantize路径下命令行输入：
+  ```
+    bash quantize_eval.sh
+  ```
+4. 窗口将输出该模型的内存占用情况、困惑度。
\ No newline at end of file

requirements.txt

+# for MiniCPM-2B hf inference
+torch>=2.0.0
+transformers==4.53.2
+gradio>=4.26.0
+
+# for vllm inference
+# vllm>=0.4.0.post1
+
+# for openai api inference
+openai>=1.17.1
+tiktoken>=0.6.0
+loguru>=0.7.2
+sentence_transformers>=2.6.1
+sse_starlette>=2.1.0
+
+# for MiniCPM-V hf inference
+Pillow>=10.3.0
+timm>=0.9.16
+sentencepiece>=0.2.0