update datasets

.codespellrc

+[codespell]
+skip = *.ipynb
+count =
+quiet-level = 3
+ignore-words-list = nd, ans, ques

configs/datasets/ARC_e/ARC_e_ppl_e6b2c5.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import ARCDataset
+
+ARC_e_reader_cfg = dict(
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
+
+ARC_e_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            "A": "Question: {question}\nAnswer: {textA}",
+            "B": "Question: {question}\nAnswer: {textB}",
+            "C": "Question: {question}\nAnswer: {textC}",
+            "D": "Question: {question}\nAnswer: {textD}"
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+ARC_e_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+ARC_e_datasets = [
+    dict(
+        type=ARCDataset,
+        abbr='ARC-e',
+        path='./data/ARC/ARC-e/ARC-Easy-Dev.jsonl',
+        reader_cfg=ARC_e_reader_cfg,
+        infer_cfg=ARC_e_infer_cfg,
+        eval_cfg=ARC_e_eval_cfg)
+]

configs/datasets/CLUE_afqmc/CLUE_afqmc_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_afqmc_ppl_c83c36 import afqmc_datasets  # noqa: F401, F403

configs/datasets/FewCLUE_bustm/FewCLUE_bustm_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .FewCLUE_bustm_ppl_47f2ab import bustm_datasets  # noqa: F401, F403

configs/datasets/FewCLUE_ocnli_fc/FewCLUE_ocnli_fc_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .FewCLUE_ocnli_fc_gen_bef37f import ocnli_fc_datasets  # noqa: F401, F403

configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_ppl_f28ad6.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+RTE_reader_cfg = dict(
+    input_columns=['hypothesis', 'premise'],
+    output_column='label',
+    test_split='train')
+
+RTE_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'entailment': '{premise}?entailment, {hypothesis}',
+            'not_entailment': '{premise}?not_entailment, {hypothesis}'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+RTE_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+RTE_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='RTE',
+        path='json',
+        data_files='./data/SuperGLUE/RTE/val.jsonl',
+        split='train',
+        reader_cfg=RTE_reader_cfg,
+        infer_cfg=RTE_infer_cfg,
+        eval_cfg=RTE_eval_cfg)
+]

configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_WiC_ppl_4118db import WiC_datasets  # noqa: F401, F403

configs/datasets/civilcomments/civilcomments_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .civilcomments_ppl_e01497 import civilcomments_datasets  # noqa: F401, F403

configs/datasets/commonsenseqa/commonsenseqa_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .commonsenseqa_gen_a58dbd import commonsenseqa_datasets  # noqa: F401, F403

configs/datasets/glm/chid.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CHIDDataset
+
+chid_reader_cfg = dict(
+    input_columns=[f'content{i}' for i in range(7)], output_column='answer')
+
+chid_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={answer: f"{{content{answer}}}"
+                  for answer in range(7)}),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+chid_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+chid_datasets = [
+    dict(
+        type=CHIDDataset,
+        path='json',
+        abbr='chid',
+        data_files='./data/FewCLUE/chid/test_public.json',
+        split='train',
+        reader_cfg=chid_reader_cfg,
+        infer_cfg=chid_infer_cfg,
+        eval_cfg=chid_eval_cfg)
+]

configs/datasets/hellaswag/hellaswag_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .hellaswag_gen_cae9cb import hellaswag_datasets  # noqa: F401, F403

configs/datasets/humaneval/humaneval_gen_d428f1.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HFDataset, HumanEvaluator
+
+humaneval_reader_cfg = dict(
+    input_columns=['prompt'], output_column='task_id', train_split='test')
+
+# TODO: allow empty output-column
+humaneval_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='Complete the following python code:\n{prompt}'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+humaneval_eval_cfg = dict(
+    evaluator=dict(type=HumanEvaluator),
+    pred_role='BOT',
+    k=[1, 10, 100],  # the parameter only for humaneval
+    pred_postprocessor=dict(type='humaneval'),
+)
+
+humaneval_datasets = [
+    dict(
+        type=HFDataset,
+        path='openai_humaneval',
+        reader_cfg=humaneval_reader_cfg,
+        infer_cfg=humaneval_infer_cfg,
+        eval_cfg=humaneval_eval_cfg)
+]

configs/datasets/iwslt2017/iwslt2017_gen_95def3.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import BM25Retriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import BleuEvaluator
+from opencompass.datasets import IWSLT2017Dataset
+
+iwslt2017_reader_cfg = dict(
+    input_columns='en', output_column='de', train_split='validation')
+
+iwslt2017_infer_cfg = dict(
+    ice_template=dict(type='PromptTemplate',
+                    template='</E>{en} = {de}',
+                    ice_token='</E>'),
+    retriever=dict(type=BM25Retriever, ice_num=1),
+    inferencer=dict(type=GenInferencer))
+
+iwslt2017_eval_cfg = dict(
+    evaluator=dict(type=BleuEvaluator), 
+    pred_role='BOT', 
+    pred_postprocessor=dict(type='general_cn'),
+    dataset_postprocessor=dict(type='general_cn'))
+
+iwslt2017_datasets = [
+    dict(
+        type=IWSLT2017Dataset,
+        path='iwslt2017',
+        name='iwslt2017-en-de',
+        reader_cfg=iwslt2017_reader_cfg,
+        infer_cfg=iwslt2017_infer_cfg,
+        eval_cfg=iwslt2017_eval_cfg)
+]
\ No newline at end of file

configs/datasets/mbpp/mbpp_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .mbpp_gen_4104e4 import mbpp_datasets  # noqa: F401, F403

configs/datasets/narrativeqa/narrativeqa_gen_5786a7.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import NarrativeQADataset, TriviaQAEvaluator
+
+narrativeqa_reader_cfg = dict(
+    input_columns=['question', 'evidence'],
+    output_column='answer',
+    train_split='valid',
+    test_split='valid')
+
+narrativeqa_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{evidence}\nAnswer these questions:\nQ: {question}?A:'),
+                dict(role='BOT', prompt=''),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(
+        type=GenInferencer, max_out_len=50, max_seq_len=8192, batch_size=4))
+
+narrativeqa_eval_cfg = dict(
+    evaluator=dict(type=TriviaQAEvaluator), pred_role='BOT')
+
+narrativeqa_datasets = [
+    dict(
+        type=NarrativeQADataset,
+        abbr='NarrativeQA',
+        path='./data/narrativeqa/',
+        reader_cfg=narrativeqa_reader_cfg,
+        infer_cfg=narrativeqa_infer_cfg,
+        eval_cfg=narrativeqa_eval_cfg)
+]

configs/datasets/qabench/qabench_gen_0d5967.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HFDataset
+
+qabench_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='reference',
+)
+
+# TODO: allow empty output-column
+qabench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[dict(role="HUMAN", prompt="{prompt}")])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+qabench_datasets = [
+    dict(
+        type=HFDataset,
+        path='csv',
+        data_files='./data/qabench/qabench-test.qa.csv',
+        abbr="qabench",
+        split='train',
+        reader_cfg=qabench_reader_cfg,
+        infer_cfg=qabench_infer_cfg,
+        eval_cfg=dict(ds_column="reference"))
+]

configs/datasets/qaspercut/qaspercut_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .qaspercut_gen_943606 import qaspercut_datasets  # noqa: F401, F403

configs/datasets/strategyqa/strategyqa_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .strategyqa_gen_be3f8d import strategyqa_datasets  # noqa: F401, F403

configs/datasets/triviaqarc/triviaqarc_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .triviaqarc_gen_6c1726 import triviaqarc_datasets  # noqa: F401, F403

configs/datasets/winogrande/winogrande_ppl_00f8ad.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import winograndeDataset
+
+winogrande_reader_cfg = dict(
+    input_columns=['opt1', 'opt2'],
+    output_column='answer',
+    train_split='validation',
+    test_split='validation')
+
+winogrande_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            i: dict(round=[
+                dict(role="HUMAN", prompt=f"Good sentence: {{opt{i+1}}}"),
+            ])
+            for i in range(2)
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+winogrande_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+winogrande_datasets = [
+    dict(
+        abbr='winogrande',
+        type=winograndeDataset,
+        path='winogrande',
+        name='winogrande_xs',
+        reader_cfg=winogrande_reader_cfg,
+        infer_cfg=winogrande_infer_cfg,
+        eval_cfg=winogrande_eval_cfg)
+]