[Feature] Add huggingface apply_chat_template (#1098)

* add TheoremQA with 5-shot * add huggingface_above_v4_33 classes * use num_worker partitioner in cli * update theoremqa * update TheoremQA * add TheoremQA * rename theoremqa -> TheoremQA * update TheoremQA output path * rewrite many model configs * update huggingface * further update * refine configs * update configs * update configs * add configs/eval_llama3_instruct.py * add summarizer multi faceted * update bbh datasets * update configs/models/hf_llama/lmdeploy_llama3_8b_instruct.py * rename class * update readme * update hf above v4.33

[Feature] Add huggingface apply_chat_template (#1098)
* add TheoremQA with 5-shot * add huggingface_above_v4_33 classes * use num_worker partitioner in cli * update theoremqa * update TheoremQA * add TheoremQA * rename theoremqa -> TheoremQA * update TheoremQA output path * rewrite many model configs * update huggingface * further update * refine configs * update configs * update configs * add configs/eval_llama3_instruct.py * add summarizer multi faceted * update bbh datasets * update configs/models/hf_llama/lmdeploy_llama3_8b_instruct.py * rename class * update readme * update hf above v4.33
7505b3ca · Fengzhe Zhou · GitHub · 6c711cb2 · 7505b3ca · 7505b3ca
Unverified Commit 7505b3ca authored May 14, 2024 by Fengzhe Zhou Committed by GitHub May 14, 2024
20 changed files
--- a/configs/models/baichuan/hf_baichuan2_7b_base.py
+++ b/configs/models/baichuan/hf_baichuan2_7b_base.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 models = [
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
        abbr='baichuan2-7b-base-hf',
-        path="baichuan-inc/Baichuan2-7B-Base",
+        path='baichuan-inc/Baichuan2-7B-Base',
-        tokenizer_path='baichuan-inc/Baichuan2-7B-Base',
+        max_out_len=1024,
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        max_out_len=100,
-        max_seq_len=2048,
        batch_size=8,
-        model_kwargs=dict(device_map='auto', trust_remote_code=True),
+        run_cfg=dict(num_gpus=1),
-        run_cfg=dict(num_gpus=1, num_procs=1),
    )
 ]
--- a/configs/models/bluelm/hf_bluelm_7b_base.py
+++ b/configs/models/bluelm/hf_bluelm_7b_base.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 models = [
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
        abbr='bluelm-7b-base-hf',
-        path="vivo-ai/BlueLM-7B-Base",
+        path='vivo-ai/BlueLM-7B-Base',
-        tokenizer_path='vivo-ai/BlueLM-7B-Base',
+        max_out_len=1024,
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        max_out_len=100,
-        max_seq_len=2048,
        batch_size=8,
-        run_cfg=dict(num_gpus=1, num_procs=1),
+        run_cfg=dict(num_gpus=1),
    )
 ]
--- a/configs/models/bluelm/hf_bluelm_7b_base_32k.py
+++ b/configs/models/bluelm/hf_bluelm_7b_base_32k.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 models = [
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
        abbr='bluelm-7b-base-32k-hf',
-        path="vivo-ai/BlueLM-7B-Base-32K",
+        path='vivo-ai/BlueLM-7B-Base-32K',
-        tokenizer_path='vivo-ai/BlueLM-7B-Base-32K',
+        max_out_len=1024,
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        max_out_len=100,
-        max_seq_len=4096,
        batch_size=8,
-        run_cfg=dict(num_gpus=1, num_procs=1),
+        run_cfg=dict(num_gpus=1),
    )
 ]
--- a/configs/models/chatglm/hf_chatglm3_6b.py
+++ b/configs/models/chatglm/hf_chatglm3_6b.py
-from opencompass.models import HuggingFaceChatGLM3
+from opencompass.models import HuggingFacewithChatTemplate
-api_meta_template = dict(
-    round=[
-        dict(role='HUMAN', api_role='HUMAN'),
-        dict(role='BOT', api_role='BOT', generate=True),
-    ]
-)
 models = [
    dict(
-        type=HuggingFaceChatGLM3,
+        type=HuggingFacewithChatTemplate,
        abbr='chatglm3-6b-hf',
        path='THUDM/chatglm3-6b',
-        tokenizer_path='THUDM/chatglm3-6b',
+        max_out_len=1024,
-        model_kwargs=dict(
+        batch_size=8,
-            device_map='auto',
+        run_cfg=dict(num_gpus=1),
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
-        meta_template=api_meta_template,
-        max_out_len=100,
-        max_seq_len=4096,
-        batch_size=1,
-        run_cfg=dict(num_gpus=1, num_procs=1)
    )
 ]
\ No newline at end of file
--- a/configs/models/chatglm/hf_chatglm3_6b_32k.py
+++ b/configs/models/chatglm/hf_chatglm3_6b_32k.py
-from opencompass.models import HuggingFaceChatGLM3
+from opencompass.models import HuggingFacewithChatTemplate
-api_meta_template = dict(
-    round=[
-        dict(role='HUMAN', api_role='HUMAN'),
-        dict(role='BOT', api_role='BOT', generate=True),
-    ]
-)
 models = [
    dict(
-        type=HuggingFaceChatGLM3,
+        type=HuggingFacewithChatTemplate,
        abbr='chatglm3-6b-32k-hf',
        path='THUDM/chatglm3-6b-32k',
-        tokenizer_path='THUDM/chatglm3-6b-32k',
+        max_out_len=1024,
-        model_kwargs=dict(
+        batch_size=8,
-            device_map='auto',
+        run_cfg=dict(num_gpus=1),
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
-        meta_template=api_meta_template,
-        max_out_len=100,
-        max_seq_len=4096,
-        batch_size=1,
-        run_cfg=dict(num_gpus=1, num_procs=1)
    )
 ]
--- a/configs/models/chatglm/hf_chatglm3_6b_base.py
+++ b/configs/models/chatglm/hf_chatglm3_6b_base.py
-from opencompass.models import HuggingFace
+from opencompass.models import HuggingFaceBaseModel
 models = [
    dict(
-        type=HuggingFace,
+        type=HuggingFaceBaseModel,
        abbr='chatglm3-6b-base-hf',
        path='THUDM/chatglm3-6b-base',
-        tokenizer_path='THUDM/chatglm3-6b-base',
+        max_out_len=1024,
-        model_kwargs=dict(
-            trust_remote_code=True,
-            device_map='auto',
-        ),
-        tokenizer_kwargs=dict(
-           padding_side='left',
-           truncation_side='left',
-           trust_remote_code=True,
-        ),
-        max_out_len=100,
-        max_seq_len=4096,
        batch_size=8,
-        run_cfg=dict(num_gpus=1, num_procs=1),
+        run_cfg=dict(num_gpus=1),
    )
 ]
--- a/configs/models/codellama/hf_codellama_13b.py
+++ b/configs/models/codellama/hf_codellama_13b.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 models = [
-    # CodeLlama 13B
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
        abbr='CodeLlama-13b',
-        path="codellama/CodeLlama-13b-hf",
+        path='codellama/CodeLlama-13b-hf',
-        tokenizer_path='codellama/CodeLlama-13b-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
        max_out_len=1024,
-        max_seq_len=2048,
        batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=1),
-        run_cfg=dict(num_gpus=2, num_procs=1),
+    )
-    ),
 ]
--- a/configs/models/codellama/hf_codellama_13b_instruct.py
+++ b/configs/models/codellama/hf_codellama_13b_instruct.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFacewithChatTemplate
 models = [
-    # CodeLlama 13B Instruct
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFacewithChatTemplate,
        abbr='CodeLlama-13b-Instruct',
-        path="codellama/CodeLlama-13b-Instruct-hf",
+        path='codellama/CodeLlama-13b-Instruct-hf',
-        tokenizer_path='codellama/CodeLlama-13b-Instruct-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
        max_out_len=1024,
-        max_seq_len=2048,
        batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=1),
-        run_cfg=dict(num_gpus=2, num_procs=1),
+    )
-    ),
 ]
--- a/configs/models/codellama/hf_codellama_13b_python.py
+++ b/configs/models/codellama/hf_codellama_13b_python.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 models = [
-    # CodeLlama 13B Python
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
        abbr='CodeLlama-13b-Python',
-        path="codellama/CodeLlama-13b-Python-hf",
+        path='codellama/CodeLlama-13b-Python-hf',
-        tokenizer_path='codellama/CodeLlama-13b-Python-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
        max_out_len=1024,
-        max_seq_len=2048,
        batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=1),
-        run_cfg=dict(num_gpus=2, num_procs=1),
+    )
-    ),
 ]
--- a/configs/models/codellama/hf_codellama_34b.py
+++ b/configs/models/codellama/hf_codellama_34b.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 models = [
-    # CodeLlama 34B
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
        abbr='CodeLlama-34b',
-        path="codellama/CodeLlama-34b-hf",
+        path='codellama/CodeLlama-34b-hf',
-        tokenizer_path='codellama/CodeLlama-34b-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
        max_out_len=1024,
-        max_seq_len=2048,
        batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=2),
-        run_cfg=dict(num_gpus=4, num_procs=1),
+    )
-    ),
 ]
--- a/configs/models/codellama/hf_codellama_34b_instruct.py
+++ b/configs/models/codellama/hf_codellama_34b_instruct.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFacewithChatTemplate
 models = [
-    # CodeLlama 34B Instruct
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFacewithChatTemplate,
        abbr='CodeLlama-34b-Instruct',
-        path="codellama/CodeLlama-34b-Instruct-hf",
+        path='codellama/CodeLlama-34b-Instruct-hf',
-        tokenizer_path='codellama/CodeLlama-34b-Instruct-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
        max_out_len=1024,
-        max_seq_len=2048,
        batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=2),
-        run_cfg=dict(num_gpus=4, num_procs=1),
+    )
-    ),
 ]
--- a/configs/models/codellama/hf_codellama_34b_python.py
+++ b/configs/models/codellama/hf_codellama_34b_python.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 models = [
-    # CodeLlama 34B Python
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
        abbr='CodeLlama-34b-Python',
-        path="codellama/CodeLlama-34b-Python-hf",
+        path='codellama/CodeLlama-34b-Python-hf',
-        tokenizer_path='codellama/CodeLlama-34b-Python-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
        max_out_len=1024,
-        max_seq_len=2048,
        batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=2),
-        run_cfg=dict(num_gpus=4, num_procs=1),
+    )
-    ),
 ]
--- a/configs/models/codellama/hf_codellama_70b.py
+++ b/configs/models/codellama/hf_codellama_70b.py
+from opencompass.models import HuggingFaceBaseModel
+models = [
+    dict(
+        type=HuggingFaceBaseModel,
+        abbr='CodeLlama-70b',
+        path='codellama/CodeLlama-70b-hf',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=4),
+    )
+]
--- a/configs/models/codellama/hf_codellama_70b_instruct.py
+++ b/configs/models/codellama/hf_codellama_70b_instruct.py
+from opencompass.models import HuggingFacewithChatTemplate
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='CodeLlama-70b-Instruct',
+        path='codellama/CodeLlama-70b-Instruct-hf',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=4),
+    )
+]
--- a/configs/models/codellama/hf_codellama_70b_python.py
+++ b/configs/models/codellama/hf_codellama_70b_python.py
+from opencompass.models import HuggingFaceBaseModel
+models = [
+    dict(
+        type=HuggingFaceBaseModel,
+        abbr='CodeLlama-70b-Python',
+        path='codellama/CodeLlama-70b-Python-hf',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=4),
+    )
+]
--- a/configs/models/codellama/hf_codellama_7b.py
+++ b/configs/models/codellama/hf_codellama_7b.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 models = [
-    # CodeLlama 7B
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
        abbr='CodeLlama-7b',
-        path="codellama/CodeLlama-7b-hf",
+        path='codellama/CodeLlama-7b-hf',
-        tokenizer_path='codellama/CodeLlama-7b-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
        max_out_len=1024,
-        max_seq_len=2048,
        batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=1),
-        run_cfg=dict(num_gpus=1, num_procs=1),
+    )
-    ),
 ]
--- a/configs/models/codellama/hf_codellama_7b_instruct.py
+++ b/configs/models/codellama/hf_codellama_7b_instruct.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFacewithChatTemplate
 models = [
-    # CodeLlama 7B Instruct
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFacewithChatTemplate,
        abbr='CodeLlama-7b-Instruct',
-        path="codellama/CodeLlama-7b-Instruct-hf",
+        path='codellama/CodeLlama-7b-Instruct-hf',
-        tokenizer_path='codellama/CodeLlama-7b-Instruct-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
        max_out_len=1024,
-        max_seq_len=2048,
        batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=1),
-        run_cfg=dict(num_gpus=1, num_procs=1),
+    )
-    ),
 ]
--- a/configs/models/codellama/hf_codellama_7b_python.py
+++ b/configs/models/codellama/hf_codellama_7b_python.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 models = [
-    # CodeLlama 7B Python
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
        abbr='CodeLlama-7b-Python',
-        path="codellama/CodeLlama-7b-Python-hf",
+        path='codellama/CodeLlama-7b-Python-hf',
-        tokenizer_path='codellama/CodeLlama-7b-Python-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
        max_out_len=1024,
-        max_seq_len=2048,
        batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        run_cfg=dict(num_gpus=1),
-        run_cfg=dict(num_gpus=1, num_procs=1),
+    )
-    ),
 ]
--- a/configs/models/deepseek/hf_deepseek_67b_base.py
+++ b/configs/models/deepseek/hf_deepseek_67b_base.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 models = [
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
        abbr='deepseek-67b-base-hf',
-        path="deepseek-ai/deepseek-llm-67b-base",
+        path='deepseek-ai/deepseek-llm-67b-base',
-        tokenizer_path='deepseek-ai/deepseek-llm-67b-base',
+        max_out_len=1024,
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        max_out_len=100,
-        max_seq_len=2048,
        batch_size=8,
-        run_cfg=dict(num_gpus=4, num_procs=1),
+        run_cfg=dict(num_gpus=4),
    )
 ]
--- a/configs/models/deepseek/hf_deepseek_67b_chat.py
+++ b/configs/models/deepseek/hf_deepseek_67b_chat.py
-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFacewithChatTemplate
-_meta_template = dict(
-    begin='<｜begin▁of▁sentence｜>',
-    round=[
-        dict(role="HUMAN", begin='User: ', end='\n\n'),
-        dict(role="BOT", begin="Assistant: ", end='<｜end▁of▁sentence｜>', generate=True),
-    ],
-)
 models = [
    dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFacewithChatTemplate,
        abbr='deepseek-67b-chat-hf',
-        path="deepseek-ai/deepseek-llm-67b-chat",
+        path='deepseek-ai/deepseek-llm-67b-chat',
-        model_kwargs=dict(
+        max_out_len=1024,
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        meta_template=_meta_template,
-        max_out_len=100,
-        max_seq_len=2048,
        batch_size=8,
-        run_cfg=dict(num_gpus=4, num_procs=1),
+        run_cfg=dict(num_gpus=4),
-        batch_padding=True,
    )
 ]