Initial commit

opencompass/configs/datasets/CHARM/charm_reason_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .charm_reason_gen_f8fca2 import charm_reason_datasets  # noqa: F401, F403

opencompass/configs/datasets/CHARM/charm_reason_gen_f8fca2.py

+import os
+from mmengine.config import read_base
+
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import CharmDataset, charm_reason_postprocess, CharmReasonEvaluator
+
+with read_base():
+    from .charm_reason_settings import charm_tasks, settings
+
+
+charm_reason_datasets = []
+
+for _cot, _cot_prefix, dataset_path, fewshot_example_path, prompt_template in settings:
+    for _task in charm_tasks:
+        _fewshot_example_file = os.path.join(fewshot_example_path, f'{_task}_{_cot}.txt')
+        with open(_fewshot_example_file, 'r') as f:
+            _hint = f.read()
+
+        charm_reason_reader_cfg = dict(input_columns=['input'], output_column='target')
+
+        charm_reason_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[dict(role='HUMAN', prompt=prompt_template.format(_hint=_hint) + _cot_prefix)]),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=512),
+        )
+
+        charm_reason_eval_cfg = dict(
+            evaluator=dict(type=CharmReasonEvaluator),
+            pred_role='BOT',
+            pred_postprocessor=dict(type=charm_reason_postprocess),
+            dataset_postprocessor=dict(type=charm_reason_postprocess),
+        )
+
+        charm_reason_datasets.append(
+            dict(
+                type=CharmDataset,
+                path=dataset_path,
+                name=_task,
+                abbr='charm-reason-' + _task + '_' + _cot,
+                reader_cfg=charm_reason_reader_cfg,
+                infer_cfg=charm_reason_infer_cfg.copy(),
+                eval_cfg=charm_reason_eval_cfg.copy(),
+            )
+        )

opencompass/configs/datasets/CHARM/charm_reason_ppl_3da4de.py

+import os
+
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.datasets import CharmDataset
+from opencompass.openicl.icl_evaluator import AccwithDetailsEvaluator
+
+charm_tasks = [
+    ['Chinese_Anachronisms_Judgment', 'AB'],
+    ['Chinese_Movie_and_Music_Recommendation', 'ABCD'],
+    ['Chinese_Natural_Language_Inference', 'ABC'],
+    ['Chinese_Reading_Comprehension', 'ABCD'],
+    ['Chinese_Sequence_Understanding', 'ABCD'],
+    ['Chinese_Sport_Understanding', 'AB'],
+    ['Chinese_Time_Understanding', 'ABCD'],
+    ['Global_Anachronisms_Judgment', 'AB'],
+    ['Global_Movie_and_Music_Recommendation', 'ABCD'],
+    ['Global_Natural_Language_Inference', 'ABC'],
+    ['Global_Reading_Comprehension', 'ABCD'],
+    ['Global_Sequence_Understanding', 'ABCD'],
+    ['Global_Sport_Understanding', 'AB'],
+    ['Global_Time_Understanding', 'ABCDEF'],
+]
+
+charm_reason_datasets = []
+for task_name, options in charm_tasks:
+
+    with open(os.path.join(os.path.dirname(__file__), 'few-shot-examples', f'{task_name}_Direct.txt'), 'r') as f:
+        few_shot_example = f.read()
+
+    charm_reason_reader_cfg = dict(input_columns=['input'], output_column='target')
+
+    charm_reason_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template={
+                f'({opt})': f'{few_shot_example}\n{{input}}\nA: {opt}' for opt in options
+            },
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=PPLInferencer),
+    )
+
+    charm_reason_eval_cfg = dict(evaluator=dict(type=AccwithDetailsEvaluator))
+
+    charm_reason_datasets.append(
+        dict(
+            type=CharmDataset,
+            abbr=f'charm-reason-{task_name}_Direct',
+            path=f'data/CHARM/reasoning',
+            name=task_name,
+            reader_cfg=charm_reason_reader_cfg,
+            infer_cfg=charm_reason_infer_cfg,
+            eval_cfg=charm_reason_eval_cfg,
+        )
+    )

opencompass/configs/datasets/CHARM/charm_reason_settings.py

+import os
+
+charm_tasks = [
+    'Chinese_Anachronisms_Judgment',
+    'Chinese_Movie_and_Music_Recommendation',
+    'Chinese_Natural_Language_Inference',
+    'Chinese_Reading_Comprehension',
+    'Chinese_Sequence_Understanding',
+    'Chinese_Sport_Understanding',
+    'Chinese_Time_Understanding',
+    'Global_Anachronisms_Judgment',
+    'Global_Movie_and_Music_Recommendation',
+    'Global_Natural_Language_Inference',
+    'Global_Reading_Comprehension',
+    'Global_Sequence_Understanding',
+    'Global_Sport_Understanding',
+    'Global_Time_Understanding',
+]
+
+XLT_template = 'Follow the given examples and answer the question.\n{_hint}\n\n I want you to act as an commonsense reasoning expert for Chinese. \n Request: {{input}}\n'
+Translate_EN_template = 'Follow the given examples and answer the question.\n{_hint}\n\nQ: {{input}}\nA: '
+Other_template = '请按照给定的例子回答问题。\n{_hint}\n\nQ：{{input}}\nA：'
+
+data_dir = 'data/CHARM'
+dataset_path_ZH = f'{data_dir}/reasoning'
+dataset_path_TransEn = f'{data_dir}/reasoning_Translate-EN'
+fewshot_example_path_ZH = os.path.join(os.path.dirname(__file__), 'few-shot-examples')
+fewshot_example_path_TransEn = os.path.join(os.path.dirname(__file__), 'few-shot-examples_Translate-EN')
+
+settings = [
+    ('Direct', '', dataset_path_ZH, fewshot_example_path_ZH, Other_template),
+    ('ZH-CoT', '让我们一步一步来思考。', dataset_path_ZH, fewshot_example_path_ZH, Other_template),
+    ('EN-CoT', "Let's think step by step.", dataset_path_ZH, fewshot_example_path_ZH, Other_template),
+    ('XLT', """You should retell the request in English.\nYou should do the answer step by step to choose the right answer.\nYou should step-by-step answer the request.\nYou should tell me the answer in this format 'So the answer is'.""", dataset_path_ZH, fewshot_example_path_ZH, XLT_template),
+    ('Translate-EN', "Let's think step by step.", dataset_path_TransEn, fewshot_example_path_TransEn, Translate_EN_template),
+]

opencompass/configs/datasets/CIBench/CIBench_generation_gen_8ab0dc.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import AgentInferencer
+
+from opencompass.datasets import CIBenchDataset, CIBenchEvaluator
+
+cibench_reader_cfg = dict(
+    input_columns=['questions'],
+    output_column='references',
+    train_split='test',
+    test_split='test')
+
+cibench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="""{questions}""",
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=AgentInferencer, infer_mode='every'),
+)
+
+libs = ['matplotlib', 'opencv', 'pandas', 'pytorch', 'scipy', 'seaborn']
+cibench_eval_cfg = dict(evaluator=dict(type=CIBenchEvaluator), pred_role='BOT')
+
+cibench_datasets = [
+    dict(
+        abbr=f'cibench_generation/{lib}',
+        type=CIBenchDataset,
+        path=f'./data/cibench_dataset/cibench_generation/{lib}',
+        internet_check=False,
+        reader_cfg=cibench_reader_cfg,
+        infer_cfg=cibench_infer_cfg,
+        eval_cfg=cibench_eval_cfg,
+    ) for lib in libs
+]

opencompass/configs/datasets/CIBench/CIBench_generation_oracle_gen_c4a7c1.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import AgentInferencer
+
+from opencompass.datasets import CIBenchDataset, CIBenchEvaluator
+
+cibench_reader_cfg = dict(
+    input_columns=['questions'],
+    output_column='references',
+    train_split='test',
+    test_split='test')
+
+cibench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="""{questions}""",
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=AgentInferencer, infer_mode='every_with_gt'),
+)
+
+libs = ['matplotlib', 'opencv', 'pandas', 'pytorch', 'scipy', 'seaborn']
+cibench_eval_cfg = dict(evaluator=dict(type=CIBenchEvaluator), pred_role='BOT')
+
+cibench_datasets = [
+    dict(
+        abbr=f'cibench_generation_oracle/{lib}',
+        type=CIBenchDataset,
+        path=f'./data/cibench_dataset/cibench_generation/{lib}',
+        internet_check=False,
+        reader_cfg=cibench_reader_cfg,
+        infer_cfg=cibench_infer_cfg,
+        eval_cfg=cibench_eval_cfg,
+    ) for lib in libs
+]

opencompass/configs/datasets/CIBench/CIBench_template_gen_e6b12a.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import AgentInferencer
+
+from opencompass.datasets import CIBenchDataset, CIBenchEvaluator
+
+cibench_reader_cfg = dict(
+    input_columns=['questions'],
+    output_column='references',
+    train_split='test',
+    test_split='test')
+
+cibench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="""{questions}""",
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=AgentInferencer, infer_mode='every'),
+)
+
+# no tensorboard
+libs = ['/lightgbm', '/matplotlib', '/nltk', '/opencv', '/pandas', '/pytorch',
+    '/scipy', '/seaborn', '/sklearn', '/tensorflow',
+    '_chinese/lightgbm', '_chinese/matplotlib', '_chinese/nltk',
+    '_chinese/opencv', '_chinese/pandas', '_chinese/pytorch',
+    '_chinese/scipy', '_chinese/seaborn', '_chinese/sklearn', '_chinese/tensorflow']
+cibench_eval_cfg = dict(evaluator=dict(type=CIBenchEvaluator), pred_role='BOT')
+cibench_datasets = [
+    dict(
+        abbr=f'cibench_template{lib}',
+        type=CIBenchDataset,
+        path=f'./data/cibench_dataset/cibench_template{lib}',
+        internet_check=False,
+        reader_cfg=cibench_reader_cfg,
+        infer_cfg=cibench_infer_cfg,
+        eval_cfg=cibench_eval_cfg,
+    ) for lib in libs
+]

opencompass/configs/datasets/CIBench/CIBench_template_oracle_gen_fecda1.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import AgentInferencer
+
+from opencompass.datasets import CIBenchDataset, CIBenchEvaluator
+
+cibench_reader_cfg = dict(
+    input_columns=['questions'],
+    output_column='references',
+    train_split='test',
+    test_split='test')
+
+cibench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="""{questions}""",
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=AgentInferencer, infer_mode='every_with_gt'),
+)
+
+# no tensorboard
+libs = ['/lightgbm', '/matplotlib', '/nltk', '/opencv', '/pandas', '/pytorch',
+    '/scipy', '/seaborn', '/sklearn', '/tensorflow',
+    '_chinese/lightgbm', '_chinese/matplotlib', '_chinese/nltk',
+    '_chinese/opencv', '_chinese/pandas', '_chinese/pytorch',
+    '_chinese/scipy', '_chinese/seaborn', '_chinese/sklearn', '_chinese/tensorflow']
+cibench_eval_cfg = dict(evaluator=dict(type=CIBenchEvaluator), pred_role='BOT')
+cibench_datasets = [
+    dict(
+        abbr=f'cibench_template_oracle{lib}',
+        type=CIBenchDataset,
+        path=f'./data/cibench_dataset/cibench_template{lib}',
+        internet_check=False,
+        reader_cfg=cibench_reader_cfg,
+        infer_cfg=cibench_infer_cfg,
+        eval_cfg=cibench_eval_cfg,
+    ) for lib in libs
+]

opencompass/configs/datasets/CLUE_C3/CLUE_C3_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_C3_gen_8c358f import C3_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_C3/CLUE_C3_gen_8c358f.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import C3Dataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+C3_reader_cfg = dict(
+    input_columns=[
+        'question',
+        'content',
+        'choice0',
+        'choice1',
+        'choice2',
+        'choice3',
+        'choices',
+    ],
+    output_column='label',
+)
+
+C3_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '{content}\n问：{question}\nA. {choice0}\nB. {choice1}\nC. {choice2}\nD. {choice3}\n请从“A”，“B”，“C”，“D”中进行选择。\n答：',
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+C3_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+C3_datasets = [
+    dict(
+        abbr='C3',
+        type=C3Dataset_V2,
+        path='./data/CLUE/C3/dev_0.json',
+        reader_cfg=C3_reader_cfg,
+        infer_cfg=C3_infer_cfg,
+        eval_cfg=C3_eval_cfg,
+    )
+]

opencompass/configs/datasets/CLUE_C3/CLUE_C3_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_C3_ppl_e24a31 import C3_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_C3/CLUE_C3_ppl_56b537.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import C3Dataset
+
+C3_reader_cfg = dict(
+    input_columns=[
+        'question', 'content', 'choice0', 'choice1', 'choice2', 'choice3',
+        'choices'
+    ],
+    output_column='label')
+
+C3_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: '文章：{content}\n问题：{question}\n答案：{choice0}',
+            1: '文章：{content}\n问题：{question}\n答案：{choice1}',
+            2: '文章：{content}\n问题：{question}\n答案：{choice2}',
+            3: '文章：{content}\n问题：{question}\n答案：{choice3}'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+C3_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+C3_datasets = [
+    dict(
+        type=C3Dataset,
+        abbr='C3',
+        path='./data/CLUE/C3/dev_0.json',
+        reader_cfg=C3_reader_cfg,
+        infer_cfg=C3_infer_cfg,
+        eval_cfg=C3_eval_cfg)
+]

opencompass/configs/datasets/CLUE_C3/CLUE_C3_ppl_e24a31.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import C3Dataset
+
+C3_reader_cfg = dict(
+    input_columns=[
+        'question', 'content', 'choice0', 'choice1', 'choice2', 'choice3',
+        'choices'
+    ],
+    output_column='label')
+
+C3_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            i: dict(round=[
+                dict(role='HUMAN', prompt='文章：{content}\n问题：{question}'),
+                dict(role='BOT', prompt=f'答案：{{choice{i}}}')
+            ])
+            for i in range(4)
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+C3_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+C3_datasets = [
+    dict(
+        type=C3Dataset,
+        abbr='C3',
+        path='./data/CLUE/C3/dev_0.json',
+        reader_cfg=C3_reader_cfg,
+        infer_cfg=C3_infer_cfg,
+        eval_cfg=C3_eval_cfg)
+]

opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_CMRC_gen_1bd3c8 import CMRC_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_1bd3c8.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import CMRCDataset, cmrc_postprocess
+
+CMRC_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+CMRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='根据文章回答问题。你的答案应该尽可能简练，请以 ‘答案是’ 开头的句式作答。\n文章：{context}\n问：{question}\n答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+CMRC_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=cmrc_postprocess),
+)
+
+CMRC_datasets = [
+    dict(
+        type=CMRCDataset,
+        abbr='CMRC_dev',
+        path='opencompass/cmrc_dev',
+        reader_cfg=CMRC_reader_cfg,
+        infer_cfg=CMRC_infer_cfg,
+        eval_cfg=CMRC_eval_cfg),
+]

opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_3749cd.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import CMRCDataset
+
+CMRC_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+CMRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(role='HUMAN', prompt='文章：{context}\n根据上文，回答如下问题：{question}'),
+            dict(role='BOT', prompt='答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+CMRC_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+)
+
+CMRC_datasets = [
+    dict(
+        type=CMRCDataset,
+        abbr='CMRC_dev',
+        path='opencompass/cmrc_dev',
+        reader_cfg=CMRC_reader_cfg,
+        infer_cfg=CMRC_infer_cfg,
+        eval_cfg=CMRC_eval_cfg),
+]

opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_8484b9.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import CMRCDataset
+
+CMRC_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+CMRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='文章：{context}\n根据上文，回答如下问题： {question}\n答：'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+CMRC_eval_cfg = dict(evaluator=dict(type=EMEvaluator), )
+
+CMRC_datasets = [
+    dict(
+        type=CMRCDataset,
+        abbr='CMRC_dev',
+        path='opencompass/cmrc_dev',
+        reader_cfg=CMRC_reader_cfg,
+        infer_cfg=CMRC_infer_cfg,
+        eval_cfg=CMRC_eval_cfg),
+]

opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_941108.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import CMRCDataset
+
+CMRC_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+CMRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='文章：{context}\n根据上文，回答如下问题：\n{question}\n答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+CMRC_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+)
+
+CMRC_datasets = [
+    dict(
+        type=CMRCDataset,
+        abbr='CMRC_dev',
+        path='opencompass/cmrc_dev',
+        reader_cfg=CMRC_reader_cfg,
+        infer_cfg=CMRC_infer_cfg,
+        eval_cfg=CMRC_eval_cfg),
+]

opencompass/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_DRCD_gen_1bd3c8 import DRCD_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_1bd3c8.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import DRCDDataset, drcd_postprocess
+
+DRCD_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+DRCD_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='根据文章回答问题。你的答案应该尽可能简练，请以 ‘答案是’ 开头的句式作答。\n文章：{context}\n问：{question}\n答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+DRCD_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=drcd_postprocess),
+
+)
+
+DRCD_datasets = [
+    dict(
+        type=DRCDDataset,
+        abbr='DRCD_dev',
+        path='opencompass/drcd_dev',
+        reader_cfg=DRCD_reader_cfg,
+        infer_cfg=DRCD_infer_cfg,
+        eval_cfg=DRCD_eval_cfg),
+]