[Feature] Add huggingface apply_chat_template (#1098)

* add TheoremQA with 5-shot

* add huggingface_above_v4_33 classes

* use num_worker partitioner in cli

* update theoremqa

* update TheoremQA

* add TheoremQA

* rename theoremqa -> TheoremQA

* update TheoremQA output path

* rewrite many model configs

* update huggingface

* further update

* refine configs

* update configs

* update configs

* add configs/eval_llama3_instruct.py

* add summarizer multi faceted

* update bbh datasets

* update configs/models/hf_llama/lmdeploy_llama3_8b_instruct.py

* rename class

* update readme

* update hf above v4.33

configs/models/baichuan/hf_baichuan2_7b_base.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
 models = [
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
         abbr='baichuan2-7b-base-hf',
-        path="baichuan-inc/Baichuan2-7B-Base",
-        tokenizer_path='baichuan-inc/Baichuan2-7B-Base',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        max_out_len=100,
-        max_seq_len=2048,
+        path='baichuan-inc/Baichuan2-7B-Base',
+        max_out_len=1024,
         batch_size=8,
-        model_kwargs=dict(device_map='auto', trust_remote_code=True),
-        run_cfg=dict(num_gpus=1, num_procs=1),
+        run_cfg=dict(num_gpus=1),
     )
 ]

configs/models/bluelm/hf_bluelm_7b_base.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
 models = [
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
         abbr='bluelm-7b-base-hf',
-        path="vivo-ai/BlueLM-7B-Base",
-        tokenizer_path='vivo-ai/BlueLM-7B-Base',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        max_out_len=100,
-        max_seq_len=2048,
+        path='vivo-ai/BlueLM-7B-Base',
+        max_out_len=1024,
         batch_size=8,
-        run_cfg=dict(num_gpus=1, num_procs=1),
+        run_cfg=dict(num_gpus=1),
     )
 ]

configs/models/bluelm/hf_bluelm_7b_base_32k.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
 models = [
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
         abbr='bluelm-7b-base-32k-hf',
-        path="vivo-ai/BlueLM-7B-Base-32K",
-        tokenizer_path='vivo-ai/BlueLM-7B-Base-32K',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        max_out_len=100,
-        max_seq_len=4096,
+        path='vivo-ai/BlueLM-7B-Base-32K',
+        max_out_len=1024,
         batch_size=8,
-        run_cfg=dict(num_gpus=1, num_procs=1),
+        run_cfg=dict(num_gpus=1),
     )
 ]

configs/models/chatglm/hf_chatglm3_6b.py

-from opencompass.models import HuggingFaceChatGLM3
-
-api_meta_template = dict(
-    round=[
-        dict(role='HUMAN', api_role='HUMAN'),
-        dict(role='BOT', api_role='BOT', generate=True),
-    ]
-)
+from opencompass.models import HuggingFacewithChatTemplate
 
 models = [
     dict(
-        type=HuggingFaceChatGLM3,
+        type=HuggingFacewithChatTemplate,
         abbr='chatglm3-6b-hf',
         path='THUDM/chatglm3-6b',
-        tokenizer_path='THUDM/chatglm3-6b',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
-        meta_template=api_meta_template,
-        max_out_len=100,
-        max_seq_len=4096,
-        batch_size=1,
-        run_cfg=dict(num_gpus=1, num_procs=1)
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
     )
 ]

configs/models/chatglm/hf_chatglm3_6b_32k.py

-from opencompass.models import HuggingFaceChatGLM3
-
-api_meta_template = dict(
-    round=[
-        dict(role='HUMAN', api_role='HUMAN'),
-        dict(role='BOT', api_role='BOT', generate=True),
-    ]
-)
+from opencompass.models import HuggingFacewithChatTemplate
 
 models = [
     dict(
-        type=HuggingFaceChatGLM3,
+        type=HuggingFacewithChatTemplate,
         abbr='chatglm3-6b-32k-hf',
         path='THUDM/chatglm3-6b-32k',
-        tokenizer_path='THUDM/chatglm3-6b-32k',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
-        meta_template=api_meta_template,
-        max_out_len=100,
-        max_seq_len=4096,
-        batch_size=1,
-        run_cfg=dict(num_gpus=1, num_procs=1)
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
     )
 ]

configs/models/chatglm/hf_chatglm3_6b_base.py

-from opencompass.models import HuggingFace
-
+from opencompass.models import HuggingFaceBaseModel
 
 models = [
     dict(
-        type=HuggingFace,
+        type=HuggingFaceBaseModel,
         abbr='chatglm3-6b-base-hf',
         path='THUDM/chatglm3-6b-base',
-        tokenizer_path='THUDM/chatglm3-6b-base',
-        model_kwargs=dict(
-            trust_remote_code=True,
-            device_map='auto',
-        ),
-        tokenizer_kwargs=dict(
-           padding_side='left',
-           truncation_side='left',
-           trust_remote_code=True,
-        ),
-        max_out_len=100,
-        max_seq_len=4096,
+        max_out_len=1024,
         batch_size=8,
-        run_cfg=dict(num_gpus=1, num_procs=1),
+        run_cfg=dict(num_gpus=1),
     )
 ]

configs/models/codellama/hf_codellama_13b.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
 models = [
-    # CodeLlama 13B
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
         abbr='CodeLlama-13b',
-        path="codellama/CodeLlama-13b-hf",
-        tokenizer_path='codellama/CodeLlama-13b-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
+        path='codellama/CodeLlama-13b-hf',
         max_out_len=1024,
-        max_seq_len=2048,
         batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
-        run_cfg=dict(num_gpus=2, num_procs=1),
-    ),
+        run_cfg=dict(num_gpus=1),
+    )
 ]

configs/models/codellama/hf_codellama_13b_instruct.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFacewithChatTemplate
 
 models = [
-    # CodeLlama 13B Instruct
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFacewithChatTemplate,
         abbr='CodeLlama-13b-Instruct',
-        path="codellama/CodeLlama-13b-Instruct-hf",
-        tokenizer_path='codellama/CodeLlama-13b-Instruct-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
+        path='codellama/CodeLlama-13b-Instruct-hf',
         max_out_len=1024,
-        max_seq_len=2048,
         batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
-        run_cfg=dict(num_gpus=2, num_procs=1),
-    ),
+        run_cfg=dict(num_gpus=1),
+    )
 ]

configs/models/codellama/hf_codellama_13b_python.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
 models = [
-    # CodeLlama 13B Python
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
         abbr='CodeLlama-13b-Python',
-        path="codellama/CodeLlama-13b-Python-hf",
-        tokenizer_path='codellama/CodeLlama-13b-Python-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
+        path='codellama/CodeLlama-13b-Python-hf',
         max_out_len=1024,
-        max_seq_len=2048,
         batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
-        run_cfg=dict(num_gpus=2, num_procs=1),
-    ),
+        run_cfg=dict(num_gpus=1),
+    )
 ]

configs/models/codellama/hf_codellama_34b.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
 models = [
-    # CodeLlama 34B
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
         abbr='CodeLlama-34b',
-        path="codellama/CodeLlama-34b-hf",
-        tokenizer_path='codellama/CodeLlama-34b-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
+        path='codellama/CodeLlama-34b-hf',
         max_out_len=1024,
-        max_seq_len=2048,
         batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
-        run_cfg=dict(num_gpus=4, num_procs=1),
-    ),
+        run_cfg=dict(num_gpus=2),
+    )
 ]

configs/models/codellama/hf_codellama_34b_instruct.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFacewithChatTemplate
 
 models = [
-    # CodeLlama 34B Instruct
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFacewithChatTemplate,
         abbr='CodeLlama-34b-Instruct',
-        path="codellama/CodeLlama-34b-Instruct-hf",
-        tokenizer_path='codellama/CodeLlama-34b-Instruct-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
+        path='codellama/CodeLlama-34b-Instruct-hf',
         max_out_len=1024,
-        max_seq_len=2048,
         batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
-        run_cfg=dict(num_gpus=4, num_procs=1),
-    ),
+        run_cfg=dict(num_gpus=2),
+    )
 ]

configs/models/codellama/hf_codellama_34b_python.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
 models = [
-    # CodeLlama 34B Python
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
         abbr='CodeLlama-34b-Python',
-        path="codellama/CodeLlama-34b-Python-hf",
-        tokenizer_path='codellama/CodeLlama-34b-Python-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
+        path='codellama/CodeLlama-34b-Python-hf',
         max_out_len=1024,
-        max_seq_len=2048,
         batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
-        run_cfg=dict(num_gpus=4, num_procs=1),
-    ),
+        run_cfg=dict(num_gpus=2),
+    )
 ]

configs/models/codellama/hf_codellama_70b.py

+from opencompass.models import HuggingFaceBaseModel
+
+models = [
+    dict(
+        type=HuggingFaceBaseModel,
+        abbr='CodeLlama-70b',
+        path='codellama/CodeLlama-70b-hf',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=4),
+    )
+]

configs/models/codellama/hf_codellama_70b_instruct.py

+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='CodeLlama-70b-Instruct',
+        path='codellama/CodeLlama-70b-Instruct-hf',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=4),
+    )
+]

configs/models/codellama/hf_codellama_70b_python.py

+from opencompass.models import HuggingFaceBaseModel
+
+models = [
+    dict(
+        type=HuggingFaceBaseModel,
+        abbr='CodeLlama-70b-Python',
+        path='codellama/CodeLlama-70b-Python-hf',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=4),
+    )
+]

configs/models/codellama/hf_codellama_7b.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
 models = [
-    # CodeLlama 7B
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
         abbr='CodeLlama-7b',
-        path="codellama/CodeLlama-7b-hf",
-        tokenizer_path='codellama/CodeLlama-7b-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
+        path='codellama/CodeLlama-7b-hf',
         max_out_len=1024,
-        max_seq_len=2048,
         batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
-        run_cfg=dict(num_gpus=1, num_procs=1),
-    ),
+        run_cfg=dict(num_gpus=1),
+    )
 ]

configs/models/codellama/hf_codellama_7b_instruct.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFacewithChatTemplate
 
 models = [
-    # CodeLlama 7B Instruct
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFacewithChatTemplate,
         abbr='CodeLlama-7b-Instruct',
-        path="codellama/CodeLlama-7b-Instruct-hf",
-        tokenizer_path='codellama/CodeLlama-7b-Instruct-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
+        path='codellama/CodeLlama-7b-Instruct-hf',
         max_out_len=1024,
-        max_seq_len=2048,
         batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
-        run_cfg=dict(num_gpus=1, num_procs=1),
-    ),
+        run_cfg=dict(num_gpus=1),
+    )
 ]

configs/models/codellama/hf_codellama_7b_python.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
 models = [
-    # CodeLlama 7B Python
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
         abbr='CodeLlama-7b-Python',
-        path="codellama/CodeLlama-7b-Python-hf",
-        tokenizer_path='codellama/CodeLlama-7b-Python-hf',
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
+        path='codellama/CodeLlama-7b-Python-hf',
         max_out_len=1024,
-        max_seq_len=2048,
         batch_size=8,
-        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
-        run_cfg=dict(num_gpus=1, num_procs=1),
-    ),
+        run_cfg=dict(num_gpus=1),
+    )
 ]

configs/models/deepseek/hf_deepseek_67b_base.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
 models = [
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFaceBaseModel,
         abbr='deepseek-67b-base-hf',
-        path="deepseek-ai/deepseek-llm-67b-base",
-        tokenizer_path='deepseek-ai/deepseek-llm-67b-base',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        max_out_len=100,
-        max_seq_len=2048,
+        path='deepseek-ai/deepseek-llm-67b-base',
+        max_out_len=1024,
         batch_size=8,
-        run_cfg=dict(num_gpus=4, num_procs=1),
+        run_cfg=dict(num_gpus=4),
     )
 ]

configs/models/deepseek/hf_deepseek_67b_chat.py

-from opencompass.models import HuggingFaceCausalLM
-
-_meta_template = dict(
-    begin='<｜begin▁of▁sentence｜>',
-    round=[
-        dict(role="HUMAN", begin='User: ', end='\n\n'),
-        dict(role="BOT", begin="Assistant: ", end='<｜end▁of▁sentence｜>', generate=True),
-    ],
-)
+from opencompass.models import HuggingFacewithChatTemplate
 
 models = [
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFacewithChatTemplate,
         abbr='deepseek-67b-chat-hf',
-        path="deepseek-ai/deepseek-llm-67b-chat",
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        meta_template=_meta_template,
-        max_out_len=100,
-        max_seq_len=2048,
+        path='deepseek-ai/deepseek-llm-67b-chat',
+        max_out_len=1024,
         batch_size=8,
-        run_cfg=dict(num_gpus=4, num_procs=1),
-        batch_padding=True,
+        run_cfg=dict(num_gpus=4),
     )
 ]