Update configs (#9)

* Update implements

* Update

configs/datasets/FewCLUE_tnews/FewCLUE_tnews_gen_b90e4a.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import TNewsDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 tnews_reader_cfg = dict(
     input_columns="sentence",
@@ -49,7 +50,7 @@ tnews_infer_cfg = dict(
 tnews_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 tnews_datasets = [

configs/datasets/SuperGLUE_AX_b/SuperGLUE_AX_b_gen_4dfefa.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import AXDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 AX_b_reader_cfg = dict(
     input_columns=["sentence1", "sentence2"],
@@ -27,7 +28,7 @@ AX_b_infer_cfg = dict(
 AX_b_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 AX_b_datasets = [

configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen_68aac7.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import AXDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 AX_g_reader_cfg = dict(
     input_columns=["hypothesis", "premise"],
@@ -27,7 +28,7 @@ AX_g_infer_cfg = dict(
 AX_g_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 AX_g_datasets = [

configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_gen_883d50.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import BoolQDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 BoolQ_reader_cfg = dict(
     input_columns=["question", "passage"],
@@ -25,7 +26,7 @@ BoolQ_infer_cfg = dict(
 BoolQ_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 BoolQ_datasets = [

configs/datasets/SuperGLUE_CB/SuperGLUE_CB_gen_854c6c.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import CBDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 CB_reader_cfg = dict(
     input_columns=["premise", "hypothesis"],
@@ -28,7 +29,7 @@ CB_infer_cfg = dict(
 CB_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 CB_datasets = [

configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_gen_91ca53.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import COPADataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 COPA_reader_cfg = dict(
     input_columns=["question", "premise", "choice1", "choice2"],
@@ -28,7 +29,7 @@ COPA_infer_cfg = dict(
 COPA_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 COPA_datasets = [

configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_gen_27071f.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import MultiRCDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 MultiRC_reader_cfg = dict(
     input_columns=["question", "text", "answer"],
@@ -27,7 +28,7 @@ MultiRC_infer_cfg = dict(
 MultiRC_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 MultiRC_datasets = [

configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_gen_68aac7.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import AXDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 RTE_reader_cfg = dict(
     input_columns=["hypothesis", "premise"],
@@ -27,7 +28,7 @@ RTE_infer_cfg = dict(
 RTE_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 RTE_datasets = [

configs/datasets/SuperGLUE_ReCoRD/SuperGLUE_ReCoRD_gen_0f7784.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import EMEvaluator
-from opencompass.datasets import ReCoRDDataset
+from opencompass.datasets import ReCoRDDataset, ReCoRD_postprocess
 
 ReCoRD_reader_cfg = dict(
     input_columns=['question', 'text'], output_column='answers')
@@ -16,7 +16,7 @@ ReCoRD_infer_cfg = dict(
     inferencer=dict(type=GenInferencer))
 
 ReCoRD_eval_cfg = dict(
-    evaluator=dict(type=EMEvaluator), pred_postprocessor=dict(type='ReCoRD'))
+    evaluator=dict(type=EMEvaluator), pred_postprocessor=dict(type=ReCoRD_postprocess))
 
 ReCoRD_datasets = [
     dict(

configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_gen.py

 from mmengine.config import read_base
 
 with read_base():
-    from .SuperGLUE_WSC_gen_6dc406 import WSC_datasets  # noqa: F401, F403
+    from .SuperGLUE_WSC_gen_8a881c import WSC_datasets  # noqa: F401, F403

configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_gen_6dc406.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import WSCDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 WSC_reader_cfg = dict(
     input_columns=["span1", "span2", "text"],
@@ -27,7 +28,7 @@ WSC_infer_cfg = dict(
 WSC_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 WSC_datasets = [

configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_gen_8a881c.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import WSCDataset_V3
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+WSC_reader_cfg = dict(
+    input_columns=["span1", "span2", "text"],
+    output_column="label",
+)
+
+WSC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role="HUMAN",
+                prompt=
+                "Passage: {text}\nDoes the pronoun # {span2} # refer to * {span1} *?\nA. Yes\nB. No\nAnseer:"
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+WSC_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role="BOT",
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+WSC_datasets = [
+    dict(
+        abbr="WSC",
+        type=WSCDataset_V3,
+        path="./data/SuperGLUE/WSC/val.jsonl",
+        reader_cfg=WSC_reader_cfg,
+        infer_cfg=WSC_infer_cfg,
+        eval_cfg=WSC_eval_cfg,
+    )
+]

configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_gen_d06864.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import WiCDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 WiC_reader_cfg = dict(
     input_columns=[
@@ -31,7 +32,7 @@ WiC_infer_cfg = dict(
 WiC_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 WiC_datasets = [

configs/datasets/TheoremQA/TheoremQA_gen.py

 from mmengine.config import read_base
 
 with read_base():
-    from .TheoremQA_gen_a27a10 import TheoremQA_datasets  # noqa: F401, F403
+    from .TheoremQA_gen_7009de import TheoremQA_datasets  # noqa: F401, F403

configs/datasets/TheoremQA/TheoremQA_gen_6365d5.py → configs/datasets/TheoremQA/TheoremQA_gen_424e0a.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
-from opencompass.datasets import TheoremQADataset
+from opencompass.datasets import TheoremQADataset, TheoremQA_postprocess
 
 TheoremQA_reader_cfg = dict(
     input_columns=['Question', 'Answer_type'],
@@ -23,11 +23,11 @@ TheoremQA_infer_cfg = dict(
         type=PromptTemplate,
         template=TheoremQA_prompt2),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer))
+    inferencer=dict(type=GenInferencer, max_out_len=512))
 
 TheoremQA_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_postprocessor=dict(type='TheoremQA'))
+    pred_postprocessor=dict(type=TheoremQA_postprocess))
 
 TheoremQA_datasets = [
     dict(

configs/datasets/TheoremQA/TheoremQA_gen_a27a10.py → configs/datasets/TheoremQA/TheoremQA_gen_7009de.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
-from opencompass.datasets import TheoremQADataset
+from opencompass.datasets import TheoremQADataset, TheoremQA_postprocess
 
 TheoremQA_reader_cfg = dict(
     input_columns=['Question', 'Answer_type'],
@@ -31,11 +31,11 @@ TheoremQA_infer_cfg = dict(
                 dict(role='HUMAN', prompt=TheoremQA_prompt2),
             ])),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer))
+    inferencer=dict(type=GenInferencer, max_out_len=512))
 
 TheoremQA_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_postprocessor=dict(type='TheoremQA'))
+    pred_postprocessor=dict(type=TheoremQA_postprocess))
 
 TheoremQA_datasets = [
     dict(

configs/datasets/TheoremQA/TheoremQA_gen_8acdf7.py → configs/datasets/TheoremQA/TheoremQA_gen_ef26ca.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
-from opencompass.datasets import TheoremQADataset
+from opencompass.datasets import TheoremQADataset, TheoremQA_postprocess
 
 TheoremQA_reader_cfg = dict(
     input_columns=['Question', 'Answer_type'],
@@ -20,11 +20,11 @@ TheoremQA_infer_cfg = dict(
             ),
         ])),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer))
+    inferencer=dict(type=GenInferencer, max_out_len=512))
 
 TheoremQA_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_postprocessor=dict(type='TheoremQA'))
+    pred_postprocessor=dict(type=TheoremQA_postprocess))
 
 TheoremQA_datasets = [
     dict(

configs/datasets/XLSum/XLSum_gen_2bb71c.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import RougeEvaluator
-from opencompass.datasets import XLSUMDataset
+from opencompass.datasets import XLSUMDataset, Xsum_postprocess
 
 XLSum_reader_cfg = dict(input_columns=['text'], output_column='summary')
 
@@ -16,7 +16,7 @@ XLSum_infer_cfg = dict(
 
 XLSum_eval_cfg = dict(
     evaluator=dict(type=RougeEvaluator),
-    pred_postprocessor=dict(type='Xsum'),
+    pred_postprocessor=dict(type=Xsum_postprocess),
 )
 
 XLSum_datasets = [

configs/datasets/Xsum/Xsum_gen_8ea5f8.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import RougeEvaluator
-from opencompass.datasets import XsumDataset
+from opencompass.datasets import XsumDataset, Xsum_postprocess
 
 Xsum_reader_cfg = dict(input_columns=['dialogue'], output_column='summary')
 
@@ -16,7 +16,7 @@ Xsum_infer_cfg = dict(
 
 Xsum_eval_cfg = dict(
     evaluator=dict(type=RougeEvaluator),
-    pred_postprocessor=dict(type='Xsum'),
+    pred_postprocessor=dict(type=Xsum_postprocess),
 )
 
 Xsum_datasets = [

configs/datasets/agieval/agieval_gen_0a9ace.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import AGIEvalDataset, AGIEvalEvaluator
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 agieval_reader_cfg = dict(
     input_columns=['problem_input'], output_column='label')
@@ -44,7 +45,7 @@ for name in agieval_single_choice_sets:
 
     agieval_eval_cfg = dict(
         evaluator=dict(type=AccEvaluator),
-        pred_postprocessor=dict(type='first-capital'))
+        pred_postprocessor=dict(type=first_capital_postprocess))
 
     agieval_datasets.append(
         dict(