Initial commit

opencompass/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_3749cd.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import DRCDDataset
+
+DRCD_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+DRCD_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(role='HUMAN', prompt='文章：{context}\n根据上文，回答如下问题：{question}'),
+            dict(role='BOT', prompt='答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+DRCD_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+)
+
+DRCD_datasets = [
+    dict(
+        type=DRCDDataset,
+        abbr='DRCD_dev',
+        path='opencompass/drcd_dev',
+        reader_cfg=DRCD_reader_cfg,
+        infer_cfg=DRCD_infer_cfg,
+        eval_cfg=DRCD_eval_cfg),
+]

opencompass/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_8484b9.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import DRCDDataset
+
+DRCD_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+DRCD_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='文章：{context}\n根据上文，回答如下问题： {question}\n答：'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+DRCD_eval_cfg = dict(evaluator=dict(type=EMEvaluator), )
+
+DRCD_datasets = [
+    dict(
+        type=DRCDDataset,
+        abbr='DRCD_dev',
+        path='opencompass/drcd_dev',
+        reader_cfg=DRCD_reader_cfg,
+        infer_cfg=DRCD_infer_cfg,
+        eval_cfg=DRCD_eval_cfg),
+]

opencompass/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_941108.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import DRCDDataset
+
+DRCD_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+DRCD_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='文章：{context}\n根据上文，回答如下问题：\n{question}\n答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+DRCD_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+)
+
+DRCD_datasets = [
+    dict(
+        type=DRCDDataset,
+        abbr='DRCD_dev',
+        path='opencompass/drcd_dev',
+        reader_cfg=DRCD_reader_cfg,
+        infer_cfg=DRCD_infer_cfg,
+        eval_cfg=DRCD_eval_cfg),
+]

opencompass/configs/datasets/CLUE_afqmc/CLUE_afqmc_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_afqmc_gen_901306 import afqmc_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_afqmc/CLUE_afqmc_gen_901306.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import AFQMCDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+afqmc_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+afqmc_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '语句一：“{sentence1}”\n语句二：“{sentence2}”\n语句一与语句二是关于蚂蚁金融产品的疑问，两者所询问的内容是否完全一致？\nA. 不完全一致\nB. 完全一致\n请从“A”，“B”中进行选择。\n答：',
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+afqmc_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+afqmc_datasets = [
+    dict(
+        abbr='afqmc-dev',
+        type=AFQMCDatasetV2,
+        path='opencompass/afqmc-dev',
+        reader_cfg=afqmc_reader_cfg,
+        infer_cfg=afqmc_infer_cfg,
+        eval_cfg=afqmc_eval_cfg,
+    ),
+]

opencompass/configs/datasets/CLUE_afqmc/CLUE_afqmc_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_afqmc_ppl_6507d7 import afqmc_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_afqmc/CLUE_afqmc_ppl_378c5b.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+afqmc_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+afqmc_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            dict(round=[
+                dict(
+                    role='HUMAN', prompt='“{sentence1}”与“{sentence2}”不同还是相似？'),
+                dict(role='BOT', prompt='不同。')
+            ]),
+            1:
+            dict(round=[
+                dict(
+                    role='HUMAN', prompt='“{sentence1}”与“{sentence2}”不同还是相似？'),
+                dict(role='BOT', prompt='相似')
+            ]),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+afqmc_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+afqmc_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='afqmc-dev',
+        path='json',
+        data_files='./data/CLUE/AFQMC/dev.json',
+        split='train',
+        reader_cfg=afqmc_reader_cfg,
+        infer_cfg=afqmc_infer_cfg,
+        eval_cfg=afqmc_eval_cfg),
+]

opencompass/configs/datasets/CLUE_afqmc/CLUE_afqmc_ppl_6507d7.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+afqmc_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+afqmc_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    '语句一：“{sentence1}”\n语句二：“{sentence2}”\n语句一与语句二是关于蚂蚁金融产品的疑问，两者所询问的内容是否完全一致？'
+                ),
+                dict(role='BOT', prompt='不完全一致')
+            ]),
+            1:
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    '语句一：“{sentence1}”\n语句二：“{sentence2}”\n语句一与语句二是关于蚂蚁金融产品的疑问，两者所询问的内容是否完全一致？'
+                ),
+                dict(role='BOT', prompt='完全一致')
+            ]),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+afqmc_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+afqmc_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='afqmc-dev',
+        path='json',
+        data_files='./data/CLUE/AFQMC/dev.json',
+        split='train',
+        reader_cfg=afqmc_reader_cfg,
+        infer_cfg=afqmc_infer_cfg,
+        eval_cfg=afqmc_eval_cfg),
+]

opencompass/configs/datasets/CLUE_afqmc/CLUE_afqmc_ppl_7b0c1e.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+afqmc_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+afqmc_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: '{sentence1},{sentence2}不同。',
+            1: '{sentence1},{sentence2}相似。'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+afqmc_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+afqmc_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='afqmc-dev',
+        path='json',
+        data_files='./data/CLUE/AFQMC/dev.json',
+        split='train',
+        reader_cfg=afqmc_reader_cfg,
+        infer_cfg=afqmc_infer_cfg,
+        eval_cfg=afqmc_eval_cfg),
+]

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_cmnli_gen_1abf97 import cmnli_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen_1abf97.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+cmnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+cmnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？\nA. 蕴含\nB. 矛盾\nC. 无关\n请从“A”，“B”，“C”中进行选择。\n答：'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+cmnli_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+cmnli_datasets = [
+    dict(
+        abbr='cmnli',
+        type=CMNLIDatasetV2,
+        path='opencompass/cmnli-dev',
+        reader_cfg=cmnli_reader_cfg,
+        infer_cfg=cmnli_infer_cfg,
+        eval_cfg=cmnli_eval_cfg,
+    )
+]

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen_51e956.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+cmnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+cmnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}\nA. 对\nB. 错\nC. 可能\n请从“A”，“B”，“C”中进行选择。\n答：'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+cmnli_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+cmnli_datasets = [
+    dict(
+        abbr='cmnli',
+        type=CMNLIDatasetV2,
+        path='opencompass/cmnli-dev',
+        reader_cfg=cmnli_reader_cfg,
+        infer_cfg=cmnli_infer_cfg,
+        eval_cfg=cmnli_eval_cfg,
+    )
+]

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_cmnli_ppl_fdc6de import cmnli_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_98dd6e.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDataset
+
+cmnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+cmnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'contradiction':
+            '阅读文章：{sentence1}\n根据上文，回答如下问题： {sentence2}？\n答：错',
+            'entailment': '阅读文章：{sentence1}\n根据上文，回答如下问题： {sentence2}？\n答：对',
+            'neutral': '如果{sentence1}为真，那么{sentence2}也为真吗?可能'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+cmnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+cmnli_datasets = [
+    dict(
+        abbr='cmnli',
+        type=CMNLIDataset,
+        path='opencompass/cmnli-dev',
+        reader_cfg=cmnli_reader_cfg,
+        infer_cfg=cmnli_infer_cfg,
+        eval_cfg=cmnli_eval_cfg)
+]

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_ef69e7.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDataset
+
+cmnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+cmnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'contradiction':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？'),
+                dict(role='BOT', prompt='错')
+            ]),
+            'entailment':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？'),
+                dict(role='BOT', prompt='对')
+            ]),
+            'neutral':
+            dict(round=[
+                dict(
+                    role='HUMAN', prompt='如果{sentence1}为真，那么{sentence2}也为真吗？'),
+                dict(role='BOT', prompt='可能')
+            ]),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+cmnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+cmnli_datasets = [
+    dict(
+        abbr='cmnli',
+        type=CMNLIDataset,
+        path='opencompass/cmnli-dev',
+        reader_cfg=cmnli_reader_cfg,
+        infer_cfg=cmnli_infer_cfg,
+        eval_cfg=cmnli_eval_cfg)
+]

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_fdc6de.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDataset
+
+cmnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+cmnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'contradiction':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？'
+                ),
+                dict(role='BOT', prompt='矛盾')
+            ]),
+            'entailment':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？'
+                ),
+                dict(role='BOT', prompt='蕴含')
+            ]),
+            'neutral':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？'
+                ),
+                dict(role='BOT', prompt='无关')
+            ]),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+cmnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+cmnli_datasets = [
+    dict(
+        abbr='cmnli',
+        type=CMNLIDataset,
+        path='opencompass/cmnli-dev',
+        reader_cfg=cmnli_reader_cfg,
+        infer_cfg=cmnli_infer_cfg,
+        eval_cfg=cmnli_eval_cfg)
+]

opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_ocnli_gen_c4cb6c import ocnli_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_gen_51e956.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+ocnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+)
+
+# TODO: two prompt templates for ocnli
+ocnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}\nA. 对\nB. 错\nC. 可能\n请从“A”，“B”，“C”中进行选择。\n答：'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ocnli_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+ocnli_datasets = [
+    dict(
+        abbr='ocnli',
+        type=CMNLIDatasetV2,  # ocnli share the same format with cmnli
+        path='opencompass/OCNLI-dev',
+        reader_cfg=ocnli_reader_cfg,
+        infer_cfg=ocnli_infer_cfg,
+        eval_cfg=ocnli_eval_cfg,
+    )
+]

opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_gen_c4cb6c.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+ocnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+)
+
+# TODO: two prompt templates for ocnli
+ocnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？\nA. 蕴含\n B. 矛盾\n C. 无关\n请从“A”，“B”，“C”中进行选择。\n答：'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ocnli_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+ocnli_datasets = [
+    dict(
+        abbr='ocnli',
+        type=CMNLIDatasetV2,  # ocnli share the same format with cmnli
+        path='opencompass/OCNLI-dev',
+        reader_cfg=ocnli_reader_cfg,
+        infer_cfg=ocnli_infer_cfg,
+        eval_cfg=ocnli_eval_cfg,
+    )
+]

opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_ocnli_ppl_fdc6de import ocnli_datasets  # noqa: F401, F403