update vllm and lmdeploy configs

README.md

@@ -69,9 +69,9 @@ python tools/list_configs.py llama mmlu
 
 1、使用vllm推理验证
 ```shell
-python run.py configs/eval_llama2_vllm.py
+python run.py configs/vllm/eval_llama2_vllm.py
 ```
-其它模型使用参考`configs/eval_xxx_vllm.py`
+其它模型使用参考`configs/vllm/eval_xxx_vllm.py`
 
 2、使用lmdeploy推理验证
 ```shell
@@ -85,7 +85,7 @@ source /opt/dtk-24.04/cuda/env.sh
 #进入opencompass进行评测
 cd opencompass 
 #fp16精度评测方法
-python run.py configs/eval_llama2_lmdeploy.py
+python run.py configs/lmdeploy/eval_llama2_lmdeploy.py
 #awq int4 评测方法
 #首先需要转换awqInt4模型  
 #model_name:模型名字如 llama2 qwen-7b 
@@ -93,13 +93,15 @@ python run.py configs/eval_llama2_lmdeploy.py
 #awq_lmdeploymodel_path:生成的lmdeploy格式 awq模型路径
 lmdeploy convert ${model_name} ${awq_modelpath} --model-format awq --group-size 128 --dst-path ${awq_lmdeploymodel_path}
 #将eval_llama2_lmdeploy.py中的Llama-2-7b-hf 替换为转换好的awq_lmdeploymodel_path
-python run.py configs/eval_llama2_lmdeploy.py
+python run.py configs/lmdeploy/eval_llama2_lmdeploy.py
 ```
+其它模型使用参考`configs/lmdeploy/eval_xxx_lmdeploy.py`
 
 3、使用tgi推理验证
 ```shell
-python run.py configs/eval_llama2_tgi.py
+python run.py configs/tgi/eval_llama2_tgi.py
 ```
+其它模型使用参考`configs/tgi/eval_xxx_tgi.py`
 
 参数说明：
 （1）数据集配置参数

configs/eval_alaya.py

@@ -8,4 +8,4 @@ with read_base():
     from .datasets.mmlu.mmlu_gen import mmlu_datasets
     from .models.alaya.alaya import models
 
-datasets = [*bbh_datasets, *ceval_datasets, *cmmlu_datasets, *agieval_datasets, *mmlu_datasets]
+datasets = [*bbh_datasets, *ceval_datasets, *cmmlu_datasets, *agieval_datasets, *mmlu_datasets]
\ No newline at end of file

configs/eval_llama2_chat_lmdeploy.py → configs/lmdeploy/eval_llama2_chat_lmdeploy.py

-from mmengine.config import read_base
-from opencompass.models.turbomind import TurboMindModel
-
-with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .summarizers.example import summarizer
-    
-datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
-work_dir = './outputs/llama2-chat/'
-
-llama_chat_meta_template = dict(
-    round=[
-        dict(role="HUMAN", begin='[INST] ', end=' [/INST]'),
-        dict(role="BOT", begin=' ', end=' ', generate=True),
-    ],
-)
-
-models = [
-dict(
-        type=TurboMindModel,
-        abbr='llama-2-7b-chat-hf-lmdeploy',
-        path="Llama-2-7b-chat-hf",
-        meta_template=llama_chat_meta_template,
-        engine_config=dict(session_len=4096,
-                           max_batch_size=32),
-        gen_config=dict(top_k=1,
-                        top_p=0.8,
-                        temperature=1.0,
-                        max_new_tokens=100),
-        max_out_len=100,
-        max_seq_len=2048,
-        batch_size=1,
-        concurrency=1,
-        run_cfg=dict(num_gpus=1, num_procs=1),
-        end_str='[INST]',
-    )
+from mmengine.config import read_base
+from opencompass.models.turbomind import TurboMindModel
+
+with read_base():
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..summarizers.example import summarizer
+    
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
+work_dir = './outputs/llama2-chat/'
+
+llama_chat_meta_template = dict(
+    round=[
+        dict(role="HUMAN", begin='[INST] ', end=' [/INST]'),
+        dict(role="BOT", begin=' ', end=' ', generate=True),
+    ],
+)
+
+models = [
+dict(
+        type=TurboMindModel,
+        abbr='llama-2-7b-chat-hf-lmdeploy',
+        path="Llama-2-7b-chat-hf",
+        meta_template=llama_chat_meta_template,
+        engine_config=dict(session_len=4096,
+                           max_batch_size=32),
+        gen_config=dict(top_k=1,
+                        top_p=0.8,
+                        temperature=1.0,
+                        max_new_tokens=100),
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=1,
+        concurrency=1,
+        run_cfg=dict(num_gpus=1, num_procs=1),
+        end_str='[INST]',
+    )
 ]
\ No newline at end of file

configs/eval_llama2_lmdeploy.py → configs/lmdeploy/eval_llama2_lmdeploy.py

@@ -2,9 +2,9 @@ from mmengine.config import read_base
 from opencompass.models.turbomind import TurboMindModel
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..summarizers.example import summarizer
     
 datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
 work_dir = './outputs/llama2/'

configs/eval_qwen_chat_lmdeploy.py → configs/lmdeploy/eval_qwen_chat_lmdeploy.py

@@ -2,9 +2,9 @@ from mmengine.config import read_base
 from opencompass.models.turbomind import TurboMindModel
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..summarizers.example import summarizer
     
 datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
 work_dir = './outputs/qwen-chat/'

configs/eval_qwen_lmdeploy.py → configs/lmdeploy/eval_qwen_lmdeploy.py

@@ -2,9 +2,9 @@ from mmengine.config import read_base
 from opencompass.models.turbomind import TurboMindModel
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..summarizers.example import summarizer
     
 datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
 work_dir = './outputs/qwen/'

configs/eval_baichuan2_chat_vllm.py → configs/vllm/eval_baichuan2_chat_vllm.py

 from mmengine.config import read_base
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
+    from ..summarizers.example import summarizer
 
 datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
 work_dir = './outputs/baichuan2-chat/'

configs/eval_chatglm3_vllm.py → configs/vllm/eval_chatglm3_vllm.py

 from mmengine.config import read_base
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..summarizers.example import summarizer
 
 datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
 work_dir = './outputs/chatglm3/'

configs/eval_llama2_chat_vllm.py → configs/vllm/eval_llama2_chat_vllm.py

 from mmengine.config import read_base
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..summarizers.example import summarizer
 
 datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
 work_dir = './outputs/llama2-chat/'

configs/eval_llama2_int4_chat_vllm.py → configs/vllm/eval_llama2_int4_chat_vllm.py

 from mmengine.config import read_base
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..summarizers.example import summarizer
 
 datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
 work_dir = './outputs/llama2-chat/'

configs/eval_llama2_vllm.py → configs/vllm/eval_llama2_vllm.py

 from mmengine.config import read_base
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..summarizers.example import summarizer
 
 datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
 work_dir = './outputs/llama2/'

configs/eval_llama3_instruct_vllm.py → configs/vllm/eval_llama3_instruct_vllm.py

 from mmengine.config import read_base
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..summarizers.example import summarizer
 
 datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
 work_dir = './outputs/llama3-instruct/'

configs/eval_qwen1.5_chat_vllm.py → configs/vllm/eval_qwen1.5_chat_vllm.py

 from mmengine.config import read_base
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
+    from ..summarizers.example import summarizer
 
 datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
 work_dir = './outputs/qwen1.5-chat/'

configs/eval_qwen1.5_int4_chat_vllm.py → configs/vllm/eval_qwen1.5_int4_chat_vllm.py

 from mmengine.config import read_base
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
+    from ..summarizers.example import summarizer
 
 datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
 work_dir = './outputs/qwen1.5-int4-chat/'

configs/eval_qwen1.5_vllm.py → configs/vllm/eval_qwen1.5_vllm.py

 from mmengine.config import read_base
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
+    from ..summarizers.example import summarizer
 
 datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
 work_dir = './outputs/qwen1.5/'

configs/eval_qwen_chat_vllm.py → configs/vllm/eval_qwen_chat_vllm.py

 from mmengine.config import read_base
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..summarizers.example import summarizer
 
 datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
 work_dir = './outputs/qwen-chat/'

configs/eval_qwen_int4_chat.py → configs/vllm/eval_qwen_int4_chat_vllm.py

 from mmengine.config import read_base
 
 with read_base():
-    from .datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
-    from .datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
-    from .summarizers.example import summarizer
+    from ..datasets.ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets 
+    from ..datasets.ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..summarizers.example import summarizer
 
 datasets = sum([v for k, v in locals().items() if k.endswith("_datasets") or k == 'datasets'], [])
 work_dir = './outputs/qwen-int4-chat/'