Update configs (#9)

* Update implements

* Update

configs/datasets/mbpp/mbpp_gen_6590b0.py

@@ -4,7 +4,7 @@ from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.datasets import MBPPDataset, MBPPEvaluator
 
 mbpp_reader_cfg = dict(
-    input_columns=['text', 'test_list'], output_column='code')
+    input_columns=['text', 'test_list'], output_column='test_list_2')
 
 mbpp_infer_cfg = dict(
     prompt_template=dict(

configs/datasets/mbpp/mbpp_gen_78c1bc.py

@@ -4,7 +4,7 @@ from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.datasets import MBPPDataset, MBPPEvaluator
 
 mbpp_reader_cfg = dict(
-    input_columns=['text', 'test_list'], output_column='code')
+    input_columns=['text', 'test_list'], output_column='test_list_2')
 
 mbpp_infer_cfg = dict(
     prompt_template=dict(

configs/datasets/mmlu/mmlu_gen_23a9a9.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import FixKRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import MMLUDataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 # None of the mmlu dataset in huggingface is correctly parsed, so we use our own dataset reader
 # Please download the dataset from https://people.eecs.berkeley.edu/~hendrycks/data.tar
@@ -33,7 +34,7 @@ mmlu_infer_cfg = dict(
 
 mmlu_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_postprocessor=dict(type='first-capital'))
+    pred_postprocessor=dict(type=first_capital_postprocess))
 
 mmlu_all_sets = [
     "college_biology",

configs/datasets/mmlu/mmlu_gen_5d1409.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import FixKRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import MMLUDataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 # None of the mmlu dataset in huggingface is correctly parsed, so we use our own dataset reader
 # Please download the dataset from https://people.eecs.berkeley.edu/~hendrycks/data.tar
@@ -107,7 +108,7 @@ for _name in mmlu_all_sets:
 
     mmlu_eval_cfg = dict(
         evaluator=dict(type=AccEvaluator),
-        pred_postprocessor=dict(type="first-capital"))
+        pred_postprocessor=dict(type=first_capital_postprocess))
 
     mmlu_datasets.append(
         dict(

configs/datasets/mmlu/mmlu_gen_79e572.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import FixKRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import MMLUDataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 # None of the mmlu dataset in huggingface is correctly parsed, so we use our own dataset reader
 # Please download the dataset from https://people.eecs.berkeley.edu/~hendrycks/data.tar
@@ -92,7 +93,7 @@ for _name in mmlu_all_sets:
 
     mmlu_eval_cfg = dict(
         evaluator=dict(type=AccEvaluator),
-        pred_postprocessor=dict(type="first-capital"),
+        pred_postprocessor=dict(type=first_capital_postprocess),
     )
 
     mmlu_datasets.append(

configs/datasets/mmlu/mmlu_gen_a484b3.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import FixKRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import MMLUDataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 # None of the mmlu dataset in huggingface is correctly parsed, so we use our own dataset reader
 # Please download the dataset from https://people.eecs.berkeley.edu/~hendrycks/data.tar
@@ -107,7 +108,7 @@ for _name in mmlu_all_sets:
 
     mmlu_eval_cfg = dict(
         evaluator=dict(type=AccEvaluator),
-        pred_postprocessor=dict(type="first-capital"))
+        pred_postprocessor=dict(type=first_capital_postprocess))
 
     mmlu_datasets.append(
         dict(

configs/datasets/nq/nq_gen_c788f6.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import NaturalQuestionDataset, NQEvaluator
+
+nq_reader_cfg = dict(
+    input_columns=['question'], output_column='answer', train_split='test')
+
+nq_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Answer these questions, your answer should be as simple as possible, start your answer with the prompt \'The answer is \'.\nQ: {question}?'),
+                dict(role='BOT', prompt='A:'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role="BOT")
+
+nq_datasets = [
+    dict(
+        type=NaturalQuestionDataset,
+        abbr='nq',
+        path='./data/nq/',
+        reader_cfg=nq_reader_cfg,
+        infer_cfg=nq_infer_cfg,
+        eval_cfg=nq_eval_cfg)
+]

configs/datasets/obqa/obqa_gen_9069e4.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import OBQADataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 _input_columns = [
     ["question_stem", "A", "B", "C", "D"],
@@ -54,7 +55,7 @@ for _i in range(2):
     obqa_eval_cfg = dict(
         evaluator=dict(type=AccEvaluator),
         pred_role="BOT",
-        pred_postprocessor=dict(type="first-capital"),
+        pred_postprocessor=dict(type=first_capital_postprocess),
     )
 
     obqa_datasets[_i]["reader_cfg"] = obqa_reader_cfg

configs/datasets/piqa/piqa_gen_1194eb.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import piqaDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 piqa_reader_cfg = dict(
     input_columns=["goal", "sol1", "sol2"],
@@ -24,7 +25,7 @@ piqa_infer_cfg = dict(
 piqa_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 piqa_datasets = [

configs/datasets/race/race_gen_69ee4f.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import RaceDataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 race_reader_cfg = dict(
     input_columns=['article', 'question', 'A', 'B', 'C', 'D'],
@@ -23,7 +24,7 @@ race_infer_cfg = dict(
 
 race_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_postprocessor=dict(type='first-capital'),
+    pred_postprocessor=dict(type=first_capital_postprocess),
     pred_role='BOT')
 
 race_datasets = [

configs/datasets/race/race_gen_9302a5.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import RaceDataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 race_reader_cfg = dict(
     input_columns=['article', 'question', 'A', 'B', 'C', 'D'],
@@ -18,7 +19,7 @@ race_infer_cfg = dict(
 
 race_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_postprocessor=dict(type='first-capital'))
+    pred_postprocessor=dict(type=first_capital_postprocess))
 
 race_datasets = [
     dict(

configs/datasets/race/race_ppl.py

 from mmengine.config import read_base
 
 with read_base():
-    from .race_ppl_ab8734 import race_datasets  # noqa: F401, F403
+    from .race_ppl_a138cd import race_datasets  # noqa: F401, F403

configs/datasets/race/race_ppl_ab8734.py → configs/datasets/race/race_ppl_a138cd.py

@@ -18,10 +18,9 @@ race_infer_cfg = dict(
                     prompt=
                     "Read the article, and answer the question by replying A, B, C or D.\n\nArticle:\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}"
                 ),
-                dict(role="BOT", prompt=ans_token),
+                dict(role="BOT", prompt=f'A: {ans}'),
             ])
-            for ans, ans_token in [["A", "{A}"], ["B", "{B}"], ["C", "{C}"],
-                                   ["D", "{D}"]]
+            for ans in ['A', 'B', 'C', 'D']
         }),
     retriever=dict(type=ZeroRetriever),
     inferencer=dict(type=PPLInferencer))

configs/datasets/realtoxicprompts/realtoxicprompts_gen_7605e4.py

@@ -27,8 +27,9 @@ realtoxicprompts_eval_cfg = dict(
 
 realtoxicprompts_datasets = [
     dict(
+        abbr='real-toxicity-prompts',
         type=RealToxicPromptsDataset,
-        path='allenai/real-toxicity-prompts',
+        path='data/realtoxicprompts/realtoxicprompts_train.arrow',
         challenging_subset=True,
         reader_cfg=realtoxicprompts_reader_cfg,
         infer_cfg=realtoxicprompts_infer_cfg,

configs/datasets/realtoxicprompts/realtoxicprompts_gen_ac723c.py

@@ -25,8 +25,9 @@ realtoxicprompts_eval_cfg = dict(
 
 realtoxicprompts_datasets = [
     dict(
+        abbr='real-toxicity-prompts',
         type=RealToxicPromptsDataset,
-        path='allenai/real-toxicity-prompts',
+        path='data/realtoxicprompts/realtoxicprompts_train.arrow',
         challenging_subset=True,
         reader_cfg=realtoxicprompts_reader_cfg,
         infer_cfg=realtoxicprompts_infer_cfg,

configs/datasets/siqa/siqa_gen_e78df3.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import siqaDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 siqa_reader_cfg = dict(
     input_columns=["context", "question", "answerA", "answerB", "answerC"],
@@ -28,7 +29,7 @@ siqa_infer_cfg = dict(
 siqa_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 siqa_datasets = [

configs/datasets/storycloze/storycloze_gen_7f656a.py

@@ -3,6 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import storyclozeDataset_V2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 storycloze_reader_cfg = dict(
     input_columns=["context", "sentence_quiz1", "sentence_quiz2"],
@@ -27,7 +28,7 @@ storycloze_infer_cfg = dict(
 storycloze_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
     pred_role="BOT",
-    pred_postprocessor=dict(type="first-capital"),
+    pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 # The original story cloze dataset and repo are not long maintaining.

configs/datasets/strategyqa/strategyqa_gen.py

 from mmengine.config import read_base
 
 with read_base():
-    from .strategyqa_gen_b3ff20 import strategyqa_datasets  # noqa: F401, F403
+    from .strategyqa_gen_1180a7 import strategyqa_datasets  # noqa: F401, F403

configs/datasets/strategyqa/strategyqa_gen_b3ff20.py → configs/datasets/strategyqa/strategyqa_gen_1180a7.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
-from opencompass.datasets import HFDataset
+from opencompass.datasets import HFDataset, strategyqa_pred_postprocess, strategyqa_dataset_postprocess
 
 strategyqa_reader_cfg = dict(
     input_columns=['question'],
@@ -23,7 +23,7 @@ strategyqa_infer_cfg = dict(
                 dict(
                     role='BOT',
                     prompt=
-                    'Hamsters are prey animals. Prey are food for predators. Thus, hamsters provide food for some animals.\nSo the answer is yes'
+                    'Hamsters are prey animals. Prey are food for predators. Thus, hamsters provide food for some animals.\nSo the answer is yes\n'
                 ),
                 dict(
                     role='HUMAN',
@@ -33,7 +33,7 @@ strategyqa_infer_cfg = dict(
                 dict(
                     role='BOT',
                     prompt=
-                    'Brooke Shields went to Princeton University. Princeton University is about as academically rigorous as the University of Pennsylvania. Thus, Brooke Shields could also succeed at the University of Pennsylvania.\nSo the answer is yes'
+                    'Brooke Shields went to Princeton University. Princeton University is about as academically rigorous as the University of Pennsylvania. Thus, Brooke Shields could also succeed at the University of Pennsylvania.\nSo the answer is yes\n'
                 ),
                 dict(
                     role='HUMAN',
@@ -43,7 +43,7 @@ strategyqa_infer_cfg = dict(
                 dict(
                     role='BOT',
                     prompt=
-                    'Hydrogen has an atomic number of 1. 1 squared is 1. There are 5 Spice Girls. Thus, Hydrogen\'s atomic number squared is less than 5.\nSo the answer is no'
+                    'Hydrogen has an atomic number of 1. 1 squared is 1. There are 5 Spice Girls. Thus, Hydrogen\'s atomic number squared is less than 5.\nSo the answer is no\n'
                 ),
                 dict(
                     role='HUMAN',
@@ -53,7 +53,7 @@ strategyqa_infer_cfg = dict(
                 dict(
                     role='BOT',
                     prompt=
-                    'College commencement ceremonies can happen in December, May, and June. December is in the winter, so there can be frost. Thus, there could be frost at some commencements.\nSo the answer is yes'
+                    'College commencement ceremonies can happen in December, May, and June. December is in the winter, so there can be frost. Thus, there could be frost at some commencements.\nSo the answer is yes\n'
                 ),
                 dict(
                     role='HUMAN',
@@ -63,7 +63,7 @@ strategyqa_infer_cfg = dict(
                 dict(
                     role='BOT',
                     prompt=
-                    'The War in Vietnam was 6 months. The gestation period for a llama is 11 months, which is more than 6 months. Thus, a llama could not give birth twice during the War in Vietnam.\nSo the answer is no'
+                    'The War in Vietnam was 6 months. The gestation period for a llama is 11 months, which is more than 6 months. Thus, a llama could not give birth twice during the War in Vietnam.\nSo the answer is no\n'
                 ),
                 dict(
                     role='HUMAN',
@@ -71,7 +71,7 @@ strategyqa_infer_cfg = dict(
                 dict(
                     role='BOT',
                     prompt=
-                    'The density of a pear is about 0.6g/cm3, which is less than water. Objects less dense than water float. Thus, a pear would float.\nSo the answer is no'
+                    'The density of a pear is about 0.6g/cm3, which is less than water. Objects less dense than water float. Thus, a pear would float.\nSo the answer is no\n'
                 ),
                 dict(role='HUMAN', prompt='Question: {question}\nAnswer:'),
             ], )),
@@ -80,8 +80,8 @@ strategyqa_infer_cfg = dict(
 
 strategyqa_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_postprocessor=dict(type='strategyqa'),
-    dataset_postprocessor=dict(type='strategyqa_dataset'))
+    pred_postprocessor=dict(type=strategyqa_pred_postprocess),
+    dataset_postprocessor=dict(type=strategyqa_dataset_postprocess))
 
 strategyqa_datasets = [
     dict(

configs/datasets/strategyqa/strategyqa_gen_934441.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
-from opencompass.datasets import HFDataset
+from opencompass.datasets import HFDataset, strategyqa_pred_postprocess, strategyqa_dataset_postprocess
 
 strategyqa_reader_cfg = dict(
     input_columns=['question'],
@@ -44,8 +44,8 @@ Q: {question}{answer}
 
 strategyqa_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_postprocessor=dict(type='strategyqa'),
-    dataset_postprocessor=dict(type='strategyqa_dataset'))
+    pred_postprocessor=dict(type=strategyqa_pred_postprocess),
+    dataset_postprocessor=dict(type=strategyqa_dataset_postprocess))
 
 strategyqa_datasets = [
     dict(