[Feature] Add huggingface apply_chat_template (#1098)

* add TheoremQA with 5-shot

* add huggingface_above_v4_33 classes

* use num_worker partitioner in cli

* update theoremqa

* update TheoremQA

* add TheoremQA

* rename theoremqa -> TheoremQA

* update TheoremQA output path

* rewrite many model configs

* update huggingface

* further update

* refine configs

* update configs

* update configs

* add configs/eval_llama3_instruct.py

* add summarizer multi faceted

* update bbh datasets

* update configs/models/hf_llama/lmdeploy_llama3_8b_instruct.py

* rename class

* update readme

* update hf above v4.33

configs/models/mistral/hf_mixtral_8x7b_v0_1.py

+from opencompass.models import HuggingFaceBaseModel
+
+models = [
+    dict(
+        type=HuggingFaceBaseModel,
+        abbr='mixtral-8x7b-v0.1-hf',
+        path='mistralai/Mixtral-8x7B-v0.1',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=4),
+    )
+]

configs/models/mistral/vllm_mistral_7b_instruct_v0_1.py

@@ -7,7 +7,6 @@ _meta_template = dict(
         dict(role="HUMAN", begin='[INST]', end='[/INST]'),
         dict(role="BOT", begin="", end='</s>', generate=True),
     ],
-    eos_token_id=2
 )
 
 models = [

configs/models/mistral/vllm_mistral_7b_instruct_v0_2.py

@@ -7,7 +7,6 @@ _meta_template = dict(
         dict(role="HUMAN", begin='[INST]', end='[/INST]'),
         dict(role="BOT", begin="", end='</s>', generate=True),
     ],
-    eos_token_id=2
 )
 
 models = [

configs/models/mistral/vllm_mistral_7b_v0_1.py

+from opencompass.models import VLLM
+
+
+models = [
+    dict(
+        type=VLLM,
+        abbr='mistral-7b-v0.1-vllm',
+        path='mistralai/Mistral-7B-v0.1',
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=32,
+        model_kwargs=dict(dtype='bfloat16'),
+        generation_kwargs=dict(temperature=0, top_p=1, max_tokens=2048, stop_token_ids=[2]),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+        stop_words=['[INST]'],
+    )
+]

configs/models/mistral/vllm_mistral_7b_v0_2.py

+from opencompass.models import VLLM
+
+
+models = [
+    dict(
+        type=VLLM,
+        abbr='mistral-7b-v0.2-vllm',
+        path='mistral-community/Mistral-7B-v0.2',
+        max_out_len=100,
+        max_seq_len=2048,
+        batch_size=32,
+        model_kwargs=dict(dtype='bfloat16'),
+        generation_kwargs=dict(temperature=0, top_p=1, max_tokens=2048, stop_token_ids=[2]),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+        stop_words=['[INST]'],
+    )
+]

configs/models/mixtral/vllm_mixtral_8x7b_instruct_v0_1.py → configs/models/mistral/vllm_mixtral_8x7b_instruct_v0_1.py

@@ -7,7 +7,6 @@ _meta_template = dict(
         dict(role="HUMAN", begin='[INST]', end='[/INST]'),
         dict(role="BOT", begin="", end='</s>', generate=True),
     ],
-    eos_token_id=2
 )
 
 models = [

configs/models/mixtral/hf_mixtral_8x22b_instruct_v0_1.py

-from opencompass.models import HuggingFaceCausalLM
-
-
-_meta_template = dict(
-    begin="<s>",
-    round=[
-        dict(role="HUMAN", begin='[INST] ', end=' [/INST]'),
-        dict(role="BOT", begin="", end='</s> ', generate=True),
-    ],
-)
-
-models = [
-    dict(
-        abbr='mixtral-8x22b-instruct-v0.1',
-        type=HuggingFaceCausalLM,
-        path='mistralai/Mixtral-8x22B-Instruct-v0.1',
-        tokenizer_path='mistralai/Mixtral-8x22B-Instruct-v0.1',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
-        meta_template=_meta_template,
-        max_out_len=100,
-        max_seq_len=2048,
-        batch_size=8,
-        run_cfg=dict(num_gpus=2, num_procs=1),
-        batch_padding=True,
-    )
-]

configs/models/mixtral/hf_mixtral_8x22b_v0_1.py

-from opencompass.models import HuggingFaceCausalLM
-
-
-models = [
-    dict(
-        abbr='mixtral-8x22b-v0.1',
-        type=HuggingFaceCausalLM,
-        path='mistralai/Mixtral-8x22B-v0.1',
-        tokenizer_path='mistralai/Mixtral-8x22B-v0.1',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
-        max_out_len=100,
-        max_seq_len=2048,
-        batch_size=8,
-        run_cfg=dict(num_gpus=2, num_procs=1),
-    )
-]

configs/models/mixtral/hf_mixtral_8x7b_instruct_v0_1.py

-from opencompass.models import HuggingFaceCausalLM
-
-
-_meta_template = dict(
-    begin="<s>",
-    round=[
-        dict(role="HUMAN", begin='[INST] ', end=' [/INST]'),
-        dict(role="BOT", begin="", end='</s> ', generate=True),
-    ],
-)
-
-models = [
-    dict(
-        abbr='mixtral-8x7b-instruct-v0.1',
-        type=HuggingFaceCausalLM,
-        path='mistralai/Mixtral-8x7B-Instruct-v0.1',
-        tokenizer_path='mistralai/Mixtral-8x7B-Instruct-v0.1',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
-        meta_template=_meta_template,
-        max_out_len=100,
-        max_seq_len=2048,
-        batch_size=8,
-        run_cfg=dict(num_gpus=2, num_procs=1),
-        batch_padding=True,
-    )
-]

configs/models/mixtral/hf_mixtral_8x7b_v0_1.py

-from opencompass.models import HuggingFaceCausalLM
-
-
-models = [
-    dict(
-        abbr='mixtral-8x7b-v0.1',
-        type=HuggingFaceCausalLM,
-        path='mistralai/Mixtral-8x7B-v0.1',
-        tokenizer_path='mistralai/Mixtral-8x7B-v0.1',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
-        max_out_len=100,
-        max_seq_len=2048,
-        batch_size=8,
-        run_cfg=dict(num_gpus=2, num_procs=1),
-    )
-]

configs/models/nanbeige/hf_nanbeige2_8b_chat.py

-from opencompass.models import HuggingFaceCausalLM
-
-_meta_template = dict(
-    begin="<|im_start|>system\n你是一个名为\"南北阁\"的人工智能助手，正在与人类用户进行交谈。你的目标是以最有帮助和最逻辑的方式回答问题，同时确保内容的安全性。你的回答中不应包含任何有害、政治化、宗教化、不道德、种族主义、非法的内容。请确保你的回答不带有社会偏见，符合社会主义价值观。如果遇到的问题无意义或事实上不连贯，请不要回答错误的内容，而是解释问题为何无效或不连贯。如果你不知道问题的答案，也请勿提供错误的信息。<|im_end|>\n",
-    round=[
-        dict(role='HUMAN', begin='<|im_start|>user\n', end='<|im_end|>\n'),
-        dict(role='BOT', begin='<|im_start|>assistant\n', end='<|im_end|>\n', generate=True),
-    ],
-)
+from opencompass.models import HuggingFacewithChatTemplate
 
 models = [
     dict(
-        type=HuggingFaceCausalLM,
+        type=HuggingFacewithChatTemplate,
         abbr='nanbeige2-8b-chat-hf',
-        path="Nanbeige/Nanbeige2-8B-Chat",
-        tokenizer_path='Nanbeige/Nanbeige2-8B-Chat',
-        model_kwargs=dict(
-            device_map='auto',
-            torch_dtype='auto',
-            trust_remote_code=True,
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='right',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        meta_template=_meta_template,
-        batch_padding=False,
-        max_out_len=100,
-        max_seq_len=4096,
+        path='Nanbeige/Nanbeige2-8B-Chat',
+        max_out_len=1024,
         batch_size=8,
-        run_cfg=dict(num_gpus=1, num_procs=1),
-        end_str='<|im_end|>',
+        run_cfg=dict(num_gpus=1),
     )
 ]

configs/models/nanbeige/hf_nanbeige_16b_base.py

-from opencompass.models import HuggingFaceCausalLM
-
-_meta_template = dict(
-    round=[
-        dict(role='HUMAN', begin='', end=''),
-        dict(role='BOT', begin='', end='\n\n', generate=True),
-    ],
-)
-
-models = [
-    dict(
-        abbr='nanbeige-16b-base-hf',
-        type=HuggingFaceCausalLM,
-        path='Nanbeige/Nanbeige-16B-Base',
-        tokenizer_path='Nanbeige/Nanbeige-16B-Base',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-            torch_dtype='auto',
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='right',
-            truncation_side='left',
-            trust_remote_code=True
-        ),
-        meta_template=_meta_template,
-        batch_padding=False,
-        max_out_len=1024,
-        max_seq_len=4096,
-        batch_size=1,
-        run_cfg=dict(num_gpus=1, num_procs=1),
-    )
-]

configs/models/nanbeige/hf_nanbeige_16b_base_32k.py

-from opencompass.models import HuggingFaceCausalLM
-
-_meta_template = dict(
-    round=[
-        dict(role='HUMAN', begin='', end=''),
-        dict(role='BOT', begin='', end='\n\n', generate=True),
-    ],
-)
-
-models = [
-    dict(
-        type=HuggingFaceCausalLM,
-        abbr='nanbeige-16b-base-32k-hf',
-        path="Nanbeige/Nanbeige-16B-Base-32K",
-        tokenizer_path='Nanbeige/Nanbeige-16B-Base-32K',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-            torch_dtype='auto',
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='right',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        meta_template=_meta_template,
-        batch_padding=False,
-        max_out_len=1024,
-        max_seq_len=8192,
-        batch_size=8,
-        run_cfg=dict(num_gpus=1, num_procs=1),
-    )
-]

configs/models/nanbeige/hf_nanbeige_16b_chat_32k.py

-from opencompass.models import HuggingFaceCausalLM
-
-_meta_template = dict(
-    round=[
-        dict(role='HUMAN', begin='### Human: \n', end='\n\n'),
-        dict(role='BOT', begin='### Assistant: ', end='</s>', generate=True),
-    ],
-)
-
-models = [
-    dict(
-        type=HuggingFaceCausalLM,
-        abbr='nanbeige-16b-chat-32k-hf',
-        path="Nanbeige/Nanbeige-16B-Chat-32K",
-        tokenizer_path='Nanbeige/Nanbeige-16B-Chat-32K',
-        model_kwargs=dict(
-            device_map='auto',
-            trust_remote_code=True,
-            torch_dtype='auto',
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-            use_fast=False,
-        ),
-        meta_template=_meta_template,
-        max_out_len=100,
-        max_seq_len=2048,
-        batch_size=8,
-        run_cfg=dict(num_gpus=2, num_procs=1),
-        end_str='</s>',
-    )
-]

configs/models/openbmb/hf_minicpm_2b_dpo_fp32.py

-from opencompass.models import HuggingFace
-
-_meta_template = dict(
-    round=[
-        dict(role="HUMAN", begin='<用户>'),
-        dict(role="BOT", begin="<AI>", generate=True),
-    ],
-)
+from opencompass.models import HuggingFacewithChatTemplate
 
 models = [
     dict(
-        type=HuggingFace,
-        abbr='minicpm-2b-dpo-hf',
+        type=HuggingFacewithChatTemplate,
+        abbr='minicpm-2b-dpo-fp32-hf',
         path='openbmb/MiniCPM-2B-dpo-fp32',
-        model_kwargs=dict(
-            trust_remote_code=True,
-            device_map='auto',
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
-        meta_template=_meta_template,
-        max_out_len=100,
-        max_seq_len=2048,
+        max_out_len=1024,
         batch_size=8,
-        run_cfg=dict(num_gpus=1, num_procs=1),
-        batch_padding=True,
+        run_cfg=dict(num_gpus=1),
     )
 ]

configs/models/openbmb/hf_minicpm_2b_sft_bf16.py

+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='minicpm-2b-sft-bf16-hf',
+        path='openbmb/MiniCPM-2B-sft-bf16',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+    )
+]

configs/models/openbmb/hf_minicpm_2b_sft_fp32.py

-from opencompass.models import HuggingFace
-
-_meta_template = dict(
-    round=[
-        dict(role="HUMAN", begin='<用户>'),
-        dict(role="BOT", begin="<AI>", generate=True),
-    ],
-)
+from opencompass.models import HuggingFacewithChatTemplate
 
 models = [
     dict(
-        type=HuggingFace,
-        abbr='minicpm-2b-sft-hf',
+        type=HuggingFacewithChatTemplate,
+        abbr='minicpm-2b-sft-fp32-hf',
         path='openbmb/MiniCPM-2B-sft-fp32',
-        model_kwargs=dict(
-            trust_remote_code=True,
-            device_map='auto',
-        ),
-        tokenizer_kwargs=dict(
-            padding_side='left',
-            truncation_side='left',
-            trust_remote_code=True,
-        ),
-        meta_template=_meta_template,
-        max_out_len=100,
-        max_seq_len=2048,
+        max_out_len=1024,
         batch_size=8,
-        run_cfg=dict(num_gpus=1, num_procs=1),
-        batch_padding=True,
+        run_cfg=dict(num_gpus=1),
     )
 ]

configs/models/opt/hf_opt_125m.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
-# OPT-125M
-opt125m = dict(
-       type=HuggingFaceCausalLM,
-       # the folowing are HuggingFaceCausalLM init parameters
-       path='facebook/opt-125m',
-       tokenizer_path='facebook/opt-125m',
-       tokenizer_kwargs=dict(
-           padding_side='left',
-           truncation_side='left',
-           proxies=None,
-           trust_remote_code=True),
-       model_kwargs=dict(device_map='auto'),
-       max_seq_len=2048,
-       # the folowing are not HuggingFaceCausalLM init parameters
-       abbr='opt125m',                # Model abbreviation
-       max_out_len=100,               # Maximum number of generated tokens
-       batch_size=128,
-       run_cfg=dict(num_gpus=1),   # Run configuration for specifying resource requirements
+models = [
+    dict(
+        type=HuggingFaceBaseModel,
+        abbr='opt-125m-hf',
+        path='facebook/opt-125m',
+        max_out_len=1024,
+        batch_size=64,
+        run_cfg=dict(num_gpus=1),
     )
-
-models = [opt125m]
+]

configs/models/opt/hf_opt_350m.py

-from opencompass.models import HuggingFaceCausalLM
+from opencompass.models import HuggingFaceBaseModel
 
-# OPT-350M
-opt350m = dict(
-       type=HuggingFaceCausalLM,
-       # the folowing are HuggingFaceCausalLM init parameters
-       path='facebook/opt-350m',
-       tokenizer_path='facebook/opt-350m',
-       tokenizer_kwargs=dict(
-           padding_side='left',
-           truncation_side='left',
-           proxies=None,
-           trust_remote_code=True),
-       model_kwargs=dict(device_map='auto'),
-       max_seq_len=2048,
-       # the folowing are not HuggingFaceCausalLM init parameters
-       abbr='opt350m',                    # Model abbreviation
-       max_out_len=100,                   # Maximum number of generated tokens          
-       batch_size=64,
-       run_cfg=dict(num_gpus=1),    # Run configuration for specifying resource requirements
+models = [
+    dict(
+        type=HuggingFaceBaseModel,
+        abbr='opt-350m-hf',
+        path='facebook/opt-350m',
+        max_out_len=1024,
+        batch_size=32,
+        run_cfg=dict(num_gpus=1),
     )
-
-models = [opt350m]
+]