Initial commit

opencompass/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_AX_g_gen_68aac7 import AX_g_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen_68aac7.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import AXDatasetV2
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+AX_g_reader_cfg = dict(
+    input_columns=['hypothesis', 'premise'],
+    output_column='label',
+)
+
+AX_g_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+AX_g_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
+)
+
+AX_g_datasets = [
+    dict(
+        abbr='AX_g',
+        type=AXDatasetV2,
+        path='./data/SuperGLUE/AX-g/AX-g.jsonl',
+        reader_cfg=AX_g_reader_cfg,
+        infer_cfg=AX_g_infer_cfg,
+        eval_cfg=AX_g_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_AX_g_ppl_66caf3 import AX_g_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_ppl_50f8f6.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+AX_g_reader_cfg = dict(
+    input_columns=['hypothesis', 'premise'],
+    output_column='label',
+    test_split='train')
+
+AX_g_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'entailment': '{premise}?entailment, {hypothesis}',
+            'not_entailment': '{premise}?not_entailment, {hypothesis}'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+AX_g_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+AX_g_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='AX_g',
+        path='json',
+        data_files='./data/SuperGLUE/AX-g/AX-g.jsonl',
+        split='train',
+        reader_cfg=AX_g_reader_cfg,
+        infer_cfg=AX_g_infer_cfg,
+        eval_cfg=AX_g_eval_cfg)
+]

opencompass/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_ppl_66caf3.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+AX_g_reader_cfg = dict(
+    input_columns=['hypothesis', 'premise'],
+    output_column='label',
+    test_split='train')
+
+AX_g_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'entailment':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    '{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?'
+                ),
+                dict(role='BOT', prompt='Yes'),
+            ]),
+            'not_entailment':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    '{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?'
+                ),
+                dict(role='BOT', prompt='No'),
+            ])
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+AX_g_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+AX_g_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='AX_g',
+        path='json',
+        data_files='./data/SuperGLUE/AX-g/AX-g.jsonl',
+        split='train',
+        reader_cfg=AX_g_reader_cfg,
+        infer_cfg=AX_g_infer_cfg,
+        eval_cfg=AX_g_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_cot_gen_1d56df.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import BoolQDatasetV2
+from opencompass.utils.text_postprocessors import (
+    first_option_postprocess,
+)
+
+QUERY_TEMPLATE = """
+Answer the following question. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of AB. Think step by step before answering.
+
+Passage: {passage}
+
+Question: {question}
+
+A. Yes
+B. NO
+
+""".strip()
+
+BoolQ_reader_cfg = dict(
+    input_columns=['question', 'passage'],
+    output_column='label',
+)
+
+BoolQ_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt=QUERY_TEMPLATE),
+            ]
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+BoolQ_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
+)
+
+BoolQ_datasets = [
+    dict(
+        abbr='BoolQ',
+        type=BoolQDatasetV2,
+        path='opencompass/boolq',
+        reader_cfg=BoolQ_reader_cfg,
+        infer_cfg=BoolQ_infer_cfg,
+        eval_cfg=BoolQ_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_few_shot_gen_ba58ea.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever, FixKRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import BoolQDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+BoolQ_reader_cfg = dict(
+    input_columns=['question', 'passage'],
+    output_column='label',
+)
+
+BoolQ_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            begin='</E>',
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{passage}\nQuestion: {question}\nA. Yes\nB. No\nAnswer:',
+                ),
+                dict(role='BOT', prompt='{label}'),
+            ],
+        ),
+        ice_token='</E>',
+    ),
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 2, 4, 6, 8]),
+    inferencer=dict(type=GenInferencer, max_out_len=50),
+)
+
+BoolQ_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+BoolQ_datasets = [
+    dict(
+        abbr='BoolQ',
+        type=BoolQDatasetV2,
+        path='opencompass/boolq',
+        reader_cfg=BoolQ_reader_cfg,
+        infer_cfg=BoolQ_infer_cfg,
+        eval_cfg=BoolQ_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_few_shot_ppl.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever, FixKRetriever
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import BoolQDatasetV2
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+
+BoolQ_reader_cfg = dict(
+    input_columns=['question', 'passage'],
+    output_column='label',
+)
+
+BoolQ_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template={
+            'B': dict(
+                round=[
+                    dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
+                    dict(role='BOT', prompt='No'),
+                ]
+            ),
+            'A': dict(
+                round=[
+                    dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
+                    dict(role='BOT', prompt='Yes'),
+                ]
+            ),
+        },
+        ice_token='</E>',
+    ),
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 2, 4, 6, 8]),
+    inferencer=dict(type=PPLInferencer, max_out_len=50),
+)
+
+BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+BoolQ_datasets = [
+    dict(
+        abbr='BoolQ',
+        type=BoolQDatasetV2,
+        path='opencompass/boolq',
+        reader_cfg=BoolQ_reader_cfg,
+        infer_cfg=BoolQ_infer_cfg,
+        eval_cfg=BoolQ_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_BoolQ_gen_883d50 import BoolQ_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_gen_883d50.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import BoolQDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+BoolQ_reader_cfg = dict(
+    input_columns=['question', 'passage'],
+    output_column='label',
+)
+
+BoolQ_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='{passage}\nQuestion: {question}\nA. Yes\nB. No\nAnswer:'),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+BoolQ_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+BoolQ_datasets = [
+    dict(
+        abbr='BoolQ',
+        type=BoolQDatasetV2,
+        path='opencompass/boolq',
+        reader_cfg=BoolQ_reader_cfg,
+        infer_cfg=BoolQ_infer_cfg,
+        eval_cfg=BoolQ_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_BoolQ_ppl_314b96 import BoolQ_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_16b1d9.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import BoolQDatasetV2
+
+BoolQ_reader_cfg = dict(
+    input_columns=['question', 'passage'],
+    output_column='label',
+)
+
+BoolQ_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'A':
+            dict(round=[
+                dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
+                dict(role='BOT', prompt='Yes'),
+            ]),
+            'B':
+            dict(round=[
+                dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
+                dict(role='BOT', prompt='No'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+BoolQ_datasets = [
+    dict(
+        abbr='BoolQ',
+        type=BoolQDatasetV2,
+        path='opencompass/boolq',
+        reader_cfg=BoolQ_reader_cfg,
+        infer_cfg=BoolQ_infer_cfg,
+        eval_cfg=BoolQ_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_314797.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import BoolQDatasetV3
+
+BoolQ_reader_cfg = dict(
+    input_columns=['question', 'passage'],
+    output_column='label',
+    test_split='train')
+
+BoolQ_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'false':
+            dict(round=[
+                dict(role='HUMAN', prompt='Passage: {passage}\nQuestion: {question}?'),
+                dict(role='BOT', prompt='Answer: No'),
+            ]),
+            'true':
+            dict(round=[
+                dict(role='HUMAN', prompt='Passage: {passage}\nQuestion: {question}?'),
+                dict(role='BOT', prompt='Answer: Yes'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+BoolQ_datasets = [
+    dict(
+        abbr='BoolQ',
+        type=BoolQDatasetV3,
+        path='opencompass/boolq',
+        reader_cfg=BoolQ_reader_cfg,
+        infer_cfg=BoolQ_infer_cfg,
+        eval_cfg=BoolQ_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_314b96.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import BoolQDataset
+
+BoolQ_reader_cfg = dict(
+    input_columns=['question', 'passage'],
+    output_column='answer',
+    test_split='train')
+
+BoolQ_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            dict(round=[
+                dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
+                dict(role='BOT', prompt='No'),
+            ]),
+            1:
+            dict(round=[
+                dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
+                dict(role='BOT', prompt='Yes'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+BoolQ_datasets = [
+    dict(
+        type=BoolQDataset,
+        abbr='BoolQ',
+        path='json',
+        data_files='opencompass/boolq',
+        split='train',
+        reader_cfg=BoolQ_reader_cfg,
+        infer_cfg=BoolQ_infer_cfg,
+        eval_cfg=BoolQ_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_4da4db.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import BoolQDataset
+
+BoolQ_reader_cfg = dict(
+    input_columns=['question', 'passage'],
+    output_column='answer',
+    test_split='train')
+
+BoolQ_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            dict(round=[
+                dict(role='HUMAN', prompt='{passage}\nQuestion: {question}'),
+                dict(role='BOT', prompt='No.'),
+            ]),
+            1:
+            dict(round=[
+                dict(role='HUMAN', prompt='{passage}\nQuestion: {question}'),
+                dict(role='BOT', prompt='Yes.'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+BoolQ_datasets = [
+    dict(
+        type=BoolQDataset,
+        abbr='BoolQ',
+        path='json',
+        data_files='opencompass/boolq',
+        split='train',
+        reader_cfg=BoolQ_reader_cfg,
+        infer_cfg=BoolQ_infer_cfg,
+        eval_cfg=BoolQ_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_9619db.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import BoolQDataset
+
+BoolQ_reader_cfg = dict(
+    input_columns=['question', 'passage'],
+    output_column='answer',
+    test_split='train')
+
+BoolQ_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: 'Passage:{passage}。\nQuestion:{question}。\nAnswer: No.',
+            1: 'Passage:{passage}。\nQuestion:{question}。\nAnswer: Yes.',
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+BoolQ_datasets = [
+    dict(
+        type=BoolQDataset,
+        abbr='BoolQ',
+        path='json',
+        data_files='opencompass/boolq',
+        split='train',
+        reader_cfg=BoolQ_reader_cfg,
+        infer_cfg=BoolQ_infer_cfg,
+        eval_cfg=BoolQ_eval_cfg)
+]

opencompass/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_CB_gen_854c6c import CB_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_gen_854c6c.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CBDatasetV2
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+CB_reader_cfg = dict(
+    input_columns=['premise', 'hypothesis'],
+    output_column='label',
+)
+
+CB_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    '{premise}\n{hypothesis}\nWhat is the relation between the two sentences?\nA. Contradiction\nB. Entailment\nC. Neutral\nAnswer:'
+                ),
+            ], ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+CB_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='ABC'),
+)
+
+CB_datasets = [
+    dict(
+        abbr='CB',
+        type=CBDatasetV2,
+        path='./data/SuperGLUE/CB/val.jsonl',
+        reader_cfg=CB_reader_cfg,
+        infer_cfg=CB_infer_cfg,
+        eval_cfg=CB_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_ppl.py

+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_CB_ppl_0143fe import CB_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_ppl_0143fe.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+CB_reader_cfg = dict(
+    input_columns=['premise', 'hypothesis'],
+    output_column='label',
+)
+
+CB_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'contradiction':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    '{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
+                ),
+                dict(role='BOT', prompt='Contradiction'),
+            ]),
+            'entailment':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    '{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
+                ),
+                dict(role='BOT', prompt='Entailment'),
+            ]),
+            'neutral':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    '{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
+                ),
+                dict(role='BOT', prompt='Neutral'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+CB_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+CB_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='CB',
+        path='json',
+        split='train',
+        data_files='./data/SuperGLUE/CB/val.jsonl',
+        reader_cfg=CB_reader_cfg,
+        infer_cfg=CB_infer_cfg,
+        eval_cfg=CB_eval_cfg,
+    )
+]