Initial commit

opencompass/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_gen_27071f.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import MultiRCDatasetV2
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+MultiRC_reader_cfg = dict(
+    input_columns=['question', 'text', 'answer'],
+    output_column='label',
+)
+
+MultiRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '{text}\nQuestion: {question}\nAnswer: {answer}\nIs it true?\nA. Yes\nB. No\nAnswer:'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+MultiRC_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
+)
+
+MultiRC_datasets = [
+    dict(
+        abbr='MultiRC',
+        type=MultiRCDatasetV2,
+        path='./data/SuperGLUE/MultiRC/val.jsonl',
+        reader_cfg=MultiRC_reader_cfg,
+        infer_cfg=MultiRC_infer_cfg,
+        eval_cfg=MultiRC_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_MultiRC_ppl_ced824 import MultiRC_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_ppl_866273.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import MultiRCDataset
+
+MultiRC_reader_cfg = dict(
+    input_columns=['question', 'text', 'answer'], output_column='label')
+
+MultiRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: 'Passage:{text}。\nQuestion:{question}。\nAnswer: {answer}. It is false.',
+            1: 'Passage:</P>。\nQuestion:{question}。\nAnswer: {answer}. It is true.',
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+MultiRC_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+MultiRC_datasets = [
+    dict(
+        type=MultiRCDataset,
+        abbr='MultiRC',
+        path='./data/SuperGLUE/MultiRC/val.jsonl',
+        reader_cfg=MultiRC_reader_cfg,
+        infer_cfg=MultiRC_infer_cfg,
+        eval_cfg=MultiRC_eval_cfg)
+]

opencompass/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_ppl_ced824.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import MultiRCDataset
+
+MultiRC_reader_cfg = dict(
+    input_columns=['question', 'text', 'answer'],
+    output_column='label',
+)
+
+MultiRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{text}\nQuestion: {question}\nAnswer: {answer}\nIs it true?'),
+                dict(role='BOT', prompt='No, it is false.'),
+            ]),
+            1:
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{text}\nQuestion: {question}\nAnswer: {answer}\nIs it true?'),
+                dict(role='BOT', prompt='Yes, it is true.'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+MultiRC_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+MultiRC_datasets = [
+    dict(
+        type=MultiRCDataset,
+        abbr='MultiRC',
+        path='./data/SuperGLUE/MultiRC/val.jsonl',
+        reader_cfg=MultiRC_reader_cfg,
+        infer_cfg=MultiRC_infer_cfg,
+        eval_cfg=MultiRC_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_RTE_gen_68aac7 import RTE_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_gen_68aac7.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import AXDatasetV2
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+RTE_reader_cfg = dict(
+    input_columns=['hypothesis', 'premise'],
+    output_column='label',
+)
+
+RTE_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+RTE_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
+)
+
+RTE_datasets = [
+    dict(
+        abbr='RTE',
+        type=AXDatasetV2,  # rte share the same format with ax
+        path='./data/SuperGLUE/RTE/val.jsonl',
+        reader_cfg=RTE_reader_cfg,
+        infer_cfg=RTE_infer_cfg,
+        eval_cfg=RTE_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_RTE_ppl_66caf3 import RTE_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_ppl_50f8f6.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+RTE_reader_cfg = dict(
+    input_columns=['hypothesis', 'premise'],
+    output_column='label',
+    test_split='train')
+
+RTE_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'entailment': '{premise}?entailment, {hypothesis}',
+            'not_entailment': '{premise}?not_entailment, {hypothesis}'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+RTE_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+RTE_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='RTE',
+        path='json',
+        data_files='./data/SuperGLUE/RTE/val.jsonl',
+        split='train',
+        reader_cfg=RTE_reader_cfg,
+        infer_cfg=RTE_infer_cfg,
+        eval_cfg=RTE_eval_cfg)
+]

opencompass/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_ppl_66caf3.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+RTE_reader_cfg = dict(
+    input_columns=['hypothesis', 'premise'],
+    output_column='label',
+    test_split='train')
+
+RTE_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'entailment':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    '{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?'
+                ),
+                dict(role='BOT', prompt='Yes'),
+            ]),
+            'not_entailment':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    '{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?'
+                ),
+                dict(role='BOT', prompt='No'),
+            ])
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+RTE_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+RTE_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='RTE',
+        path='json',
+        data_files='./data/SuperGLUE/RTE/val.jsonl',
+        split='train',
+        reader_cfg=RTE_reader_cfg,
+        infer_cfg=RTE_infer_cfg,
+        eval_cfg=RTE_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_ReCoRD/SuperGLUE_ReCoRD_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_ReCoRD_gen_30dea0 import ReCoRD_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_ReCoRD/SuperGLUE_ReCoRD_gen_0f7784.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import ReCoRDDataset, ReCoRD_postprocess
+
+ReCoRD_reader_cfg = dict(
+    input_columns=['question', 'text'], output_column='answers')
+
+ReCoRD_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=
+        'Passage:{text}\nResult:{question}\nQuestion: What entity does ____ refer to in the result?Give me the entity name:'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+ReCoRD_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator), pred_postprocessor=dict(type=ReCoRD_postprocess))
+
+ReCoRD_datasets = [
+    dict(
+        type=ReCoRDDataset,
+        abbr='ReCoRD',
+        path='./data/SuperGLUE/ReCoRD/val.jsonl',
+        reader_cfg=ReCoRD_reader_cfg,
+        infer_cfg=ReCoRD_infer_cfg,
+        eval_cfg=ReCoRD_eval_cfg)
+]

opencompass/configs/datasets/SuperGLUE_ReCoRD/SuperGLUE_ReCoRD_gen_30dea0.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import ReCoRDDataset
+
+ReCoRD_reader_cfg = dict(
+    input_columns=['question', 'text'],
+    output_column='answers',
+)
+
+ReCoRD_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                'Passage: {text}\nResult: {question}\nQuestion: What entity does ____ refer to in the result? Give me the entity name:'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ReCoRD_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type='ReCoRD'),
+)
+
+ReCoRD_datasets = [
+    dict(
+        type=ReCoRDDataset,
+        abbr='ReCoRD',
+        path='./data/SuperGLUE/ReCoRD/val.jsonl',
+        reader_cfg=ReCoRD_reader_cfg,
+        infer_cfg=ReCoRD_infer_cfg,
+        eval_cfg=ReCoRD_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_ReCoRD/SuperGLUE_ReCoRD_gen_a69961.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import ReCoRDDatasetV2, ReCoRD_postprocess
+
+ReCoRD_reader_cfg = dict(
+    input_columns=['question', 'text'], output_column='answers')
+
+ReCoRD_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN', prompt='Passage:\n{text}\nResult:\n{question}\nQuestion:\nWhat entity does ____ refer to in the Result?\nAnswer:'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+ReCoRD_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=ReCoRD_postprocess))
+
+ReCoRD_datasets = [
+    dict(
+        type=ReCoRDDatasetV2,
+        abbr='ReCoRD',
+        path='./data/SuperGLUE/ReCoRD/val.jsonl',
+        reader_cfg=ReCoRD_reader_cfg,
+        infer_cfg=ReCoRD_infer_cfg,
+        eval_cfg=ReCoRD_eval_cfg)
+]

opencompass/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_WSC_gen_fe4bf3 import WSC_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_gen_7902a7.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import WSCDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+WSC_reader_cfg = dict(
+    input_columns=['span1', 'span2', 'text'],
+    output_column='label',
+)
+
+WSC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                "{text}\nIs '{span1}' and '{span2}' refers to the same entity in the above sentence?\nA. Yes\nB. No\nAnswer:"
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+WSC_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+WSC_datasets = [
+    dict(
+        abbr='WSC',
+        type=WSCDatasetV2,
+        path='./data/SuperGLUE/WSC/val.jsonl',
+        reader_cfg=WSC_reader_cfg,
+        infer_cfg=WSC_infer_cfg,
+        eval_cfg=WSC_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_gen_fe4bf3.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import WSCDatasetV3
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+WSC_reader_cfg = dict(
+    input_columns=['span1', 'span2', 'text'],
+    output_column='label',
+)
+
+WSC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                'Passage: {text}\nDoes the pronoun # {span2} # refer to * {span1} *?\nA. Yes\nB. No\nAnswer:'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+WSC_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+WSC_datasets = [
+    dict(
+        abbr='WSC',
+        type=WSCDatasetV3,
+        path='./data/SuperGLUE/WSC/val.jsonl',
+        reader_cfg=WSC_reader_cfg,
+        infer_cfg=WSC_infer_cfg,
+        eval_cfg=WSC_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_WSC_ppl_1c4a90 import WSC_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl_003529.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import WSCDataset
+
+WSC_reader_cfg = dict(
+    input_columns=['span1', 'span2', 'text', 'new_text'],
+    output_column='answer',
+)
+
+WSC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: dict(round=[
+                dict(role='HUMAN', prompt='{text}'),
+            ]),
+            1: dict(round=[
+                dict(role='HUMAN', prompt='{new_text}'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+WSC_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+WSC_datasets = [
+    dict(
+        type=WSCDataset,
+        path='json',
+        abbr='WSC',
+        data_files='./data/SuperGLUE/WSC/val.jsonl',
+        split='train',
+        reader_cfg=WSC_reader_cfg,
+        infer_cfg=WSC_infer_cfg,
+        eval_cfg=WSC_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl_1c4a90.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import WSCDatasetV3
+
+WSC_reader_cfg = dict(
+    input_columns=['span1', 'span2', 'text'],
+    output_column='label',
+)
+
+WSC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'A':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Passage: {text}\nDoes the pronoun # {span2} # refer to * {span1} *?\nA. Yes\nB. No\nAnswer: '
+                ),
+                dict(role='BOT', prompt='A'),
+            ]),
+            'B':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Passage: {text}\nDoes the pronoun # {span2} # refer to * {span1} *?\nA. Yes\nB. No\nAnswer: '
+                ),
+                dict(role='BOT', prompt='B'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+WSC_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+WSC_datasets = [
+    dict(
+        abbr='WSC',
+        type=WSCDatasetV3,
+        path='./data/SuperGLUE/WSC/val.jsonl',
+        reader_cfg=WSC_reader_cfg,
+        infer_cfg=WSC_infer_cfg,
+        eval_cfg=WSC_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl_d0f531.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import WSCDatasetV2
+
+WSC_reader_cfg = dict(
+    input_columns=['span1', 'span2', 'text'],
+    output_column='label',
+)
+
+WSC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'A':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    "{text}\nIs '{span1}' and '{span2}' refers to the same entity in the above sentence?"
+                ),
+                dict(role='BOT', prompt='Yes'),
+            ]),
+            'B':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    "{text}\nIs '{span1}' and '{span2}' refers to the same entity in the above sentence?"
+                ),
+                dict(role='BOT', prompt='No'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+WSC_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+WSC_datasets = [
+    dict(
+        abbr='WSC',
+        type=WSCDatasetV2,
+        path='./data/SuperGLUE/WSC/val.jsonl',
+        reader_cfg=WSC_reader_cfg,
+        infer_cfg=WSC_infer_cfg,
+        eval_cfg=WSC_eval_cfg,
+    )
+]