[Format] Add config lints (#892)

configs/datasets/OpenFinData/OpenFinData_gen_46dedb.py

@@ -18,80 +18,80 @@ OpenFinData_KW_eval_cfg = dict(evaluator=dict(type=OpenFinDataKWEvaluator))
 for _name in OpenFinData_all_list:
     if _name in OpenFinData_3choices_list:
         OpenFinData_infer_cfg = dict(
-            ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
-                        dict(role="HUMAN", prompt=f"{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\n答案: "),
-                        dict(role="BOT", prompt="{answer}")]),
-                        ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
+            ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
+                        dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\n答案: '),
+                        dict(role='BOT', prompt='{answer}')]),
+                        ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
 
         OpenFinData_datasets.append(
             dict(
                 type=OpenFinDataDataset,
-                path="./data/openfindata_release",
+                path='./data/openfindata_release',
                 name=_name,
-                abbr="OpenFinData-" + _name,
+                abbr='OpenFinData-' + _name,
                 reader_cfg=dict(
-                    input_columns=["question", "A", "B", "C"],
-                    output_column="answer"),
+                    input_columns=['question', 'A', 'B', 'C'],
+                    output_column='answer'),
                 infer_cfg=OpenFinData_infer_cfg,
                 eval_cfg=OpenFinData_eval_cfg,
             ))
 
     if _name in OpenFinData_4choices_list:
         OpenFinData_infer_cfg = dict(
-            ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
-                        dict(role="HUMAN", prompt=f"{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: "),
-                        dict(role="BOT", prompt="{answer}")]),
-                        ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
+            ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
+                        dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: '),
+                        dict(role='BOT', prompt='{answer}')]),
+                        ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
 
         OpenFinData_datasets.append(
             dict(
                 type=OpenFinDataDataset,
-                path="./data/openfindata_release",
+                path='./data/openfindata_release',
                 name=_name,
-                abbr="OpenFinData-" + _name,
+                abbr='OpenFinData-' + _name,
                 reader_cfg=dict(
-                    input_columns=["question", "A", "B", "C", "D"],
-                    output_column="answer"),
+                    input_columns=['question', 'A', 'B', 'C', 'D'],
+                    output_column='answer'),
                 infer_cfg=OpenFinData_infer_cfg,
                 eval_cfg=OpenFinData_eval_cfg,
             ))
 
     if _name in OpenFinData_5choices_list:
         OpenFinData_infer_cfg = dict(
-            ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
-                        dict(role="HUMAN", prompt=f"{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n答案: "),
-                        dict(role="BOT", prompt="{answer}")]),
-                        ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
+            ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
+                        dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n答案: '),
+                        dict(role='BOT', prompt='{answer}')]),
+                        ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
 
         OpenFinData_datasets.append(
             dict(
                 type=OpenFinDataDataset,
-                path="./data/openfindata_release",
+                path='./data/openfindata_release',
                 name=_name,
-                abbr="OpenFinData-" + _name,
+                abbr='OpenFinData-' + _name,
                 reader_cfg=dict(
-                    input_columns=["question", "A", "B", "C", "D", "E"],
-                    output_column="answer"),
+                    input_columns=['question', 'A', 'B', 'C', 'D', 'E'],
+                    output_column='answer'),
                 infer_cfg=OpenFinData_infer_cfg,
                 eval_cfg=OpenFinData_eval_cfg,
             ))
 
     if _name in OpenFinData_keyword_list:
         OpenFinData_infer_cfg = dict(
-            ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
-                        dict(role="HUMAN", prompt=f"{{question}}\n答案: "),
-                        dict(role="BOT", prompt="{answer}")]),
-                        ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
+            ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
+                        dict(role='HUMAN', prompt=f'{{question}}\n答案: '),
+                        dict(role='BOT', prompt='{answer}')]),
+                        ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
 
         OpenFinData_datasets.append(
             dict(
                 type=OpenFinDataDataset,
-                path="./data/openfindata_release",
+                path='./data/openfindata_release',
                 name=_name,
-                abbr="OpenFinData-" + _name,
+                abbr='OpenFinData-' + _name,
                 reader_cfg=dict(
-                    input_columns=["question"],
-                    output_column="answer"),
+                    input_columns=['question'],
+                    output_column='answer'),
                 infer_cfg=OpenFinData_infer_cfg,
                 eval_cfg=OpenFinData_KW_eval_cfg,
             ))

configs/datasets/PJExam/PJExam_gen_8cd97c.py

@@ -8,45 +8,45 @@ for _name in [
         'gk-2022-v1', 'gk-2022-v1-math', 'gk-2023-v1', 'gk-2023-v1-math',
         'gk-2023-v2', 'gk-2023-v2-math', 'zk-2022-v1'
 ]:
-    _hint = "请你做一道</major>选择题\n请你一步一步思考并将思考过程写在【解析】和<eoe>之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和<eoa>之间。\n例如：【答案】A<eoa>\n完整的题目回答的格式如下：\n【解析】...<eoe>\n【答案】...<eoa>\n请你严格按照上述格式作答。\n题目如下：\n"
+    _hint = '请你做一道</major>选择题\n请你一步一步思考并将思考过程写在【解析】和<eoe>之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和<eoa>之间。\n例如：【答案】A<eoa>\n完整的题目回答的格式如下：\n【解析】...<eoe>\n【答案】...<eoa>\n请你严格按照上述格式作答。\n题目如下：\n'
     _reader_cfg = {
-        "input_columns": ['question'],
-        "output_column": 'std_ans',
+        'input_columns': ['question'],
+        'output_column': 'std_ans',
     },
     _infer_cfg = {
-        "ice_template": {
-            "type": PromptTemplate,
-            "template": {
-                "round": [{
-                    "role": "HUMAN",
-                    "prompt": _hint + "{question}",
+        'ice_template': {
+            'type': PromptTemplate,
+            'template': {
+                'round': [{
+                    'role': 'HUMAN',
+                    'prompt': _hint + '{question}',
                 }]
             },
-            "ice_token": "</E>"
+            'ice_token': '</E>'
         },
-        "retriever": {
-            "type": ZeroRetriever
+        'retriever': {
+            'type': ZeroRetriever
         },
-        "inferencer": {
-            "type": GenInferencer,
-            "max_out_len": 1024,
+        'inferencer': {
+            'type': GenInferencer,
+            'max_out_len': 1024,
         }
     }
     _eval_cfg = {
-        "evaluator": {
-            "type": PJExamEvaluator
+        'evaluator': {
+            'type': PJExamEvaluator
         },
-        "pred_role": "BOT",
-        "ds_column": "eval_infos"
+        'pred_role': 'BOT',
+        'ds_column': 'eval_infos'
     }
     _dataset = {
-        "type": PJExamDataset,
-        "abbr": "PJExamDataset-" + _name,
-        "path": './data/PJExam',
-        "name": _name,
-        "reader_cfg": _reader_cfg,
-        "infer_cfg": _infer_cfg,
-        "eval_cfg": _eval_cfg,
+        'type': PJExamDataset,
+        'abbr': 'PJExamDataset-' + _name,
+        'path': './data/PJExam',
+        'name': _name,
+        'reader_cfg': _reader_cfg,
+        'infer_cfg': _infer_cfg,
+        'eval_cfg': _eval_cfg,
     }
 
     PJExam_datasets.append(_dataset)

configs/datasets/QuALITY/QuALITY_gen_c407cb.py

@@ -14,9 +14,9 @@ QuALITY_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(round=[
             dict(
-                role="HUMAN",
+                role='HUMAN',
                 prompt=
-                "Read the article, and answer the question.\n\nArticle:\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}"
+                'Read the article, and answer the question.\n\nArticle:\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}'
             ),
         ])),
     retriever=dict(type=ZeroRetriever),

configs/datasets/SVAMP/svamp_gen_fb25e4.py

@@ -9,13 +9,13 @@ svamp_infer_cfg = dict(
         template=dict(
             round=[
                 dict(role='HUMAN', prompt="Question: There are 87 oranges and 290 bananas in Philip's collection. If the bananas are organized into 2 groups and oranges are organized into 93 groups How big is each group of bananas?\nLet's think step by step\nAnswer:"),
-                dict(role='BOT', prompt="To find the size of each group of bananas, we divide the total number of bananas (290) by the number of groups (2): 290 / 2 = 145. Therefore, each group of bananas contains 145 bananas. The answer is 145.\n"),
+                dict(role='BOT', prompt='To find the size of each group of bananas, we divide the total number of bananas (290) by the number of groups (2): 290 / 2 = 145. Therefore, each group of bananas contains 145 bananas. The answer is 145.\n'),
                 dict(role='HUMAN', prompt="Question: Marco and his dad went strawberry picking. Marco's dad's strawberries weighed 11 pounds. If together their strawberries weighed 30 pounds. How much did Marco's strawberries weigh?\nLet's think step by step\nAnswer:"),
                 dict(role='BOT', prompt="To find Marco's strawberries' weight, we subtract his dad's strawberries' weight (11 pounds) from the total weight of their strawberries (30 pounds): 30 - 11 = 19. Therefore, Marco's strawberries weighed 19 pounds. The answer is 19.\n"),
                 dict(role='HUMAN', prompt="Question: Edward spent $ 6 to buy 2 books each book costing him the same amount of money. Now he has $ 12. How much did each book cost?\nLet's think step by step\nAnswer:"),
-                dict(role='BOT', prompt="To find the cost of each book, we subtract the initial amount of money Edward had ($6) from the current amount of money he has ($12) and divide it by the number of books (2): (12 - 6) / 2 = 6 / 2 = 3 Therefore, each book cost $3. The answer is 3.\n"),
+                dict(role='BOT', prompt='To find the cost of each book, we subtract the initial amount of money Edward had ($6) from the current amount of money he has ($12) and divide it by the number of books (2): (12 - 6) / 2 = 6 / 2 = 3 Therefore, each book cost $3. The answer is 3.\n'),
                 dict(role='HUMAN', prompt="Question: Frank was reading through his favorite book. The book had 3 chapters, each with the same number of pages. It has a total of 594 pages. It took Frank 607 days to finish the book. How many pages are in each chapter?\nLet's think step by step\nAnswer:"),
-                dict(role='BOT', prompt="To find the number of pages in each chapter, we divide the total number of pages in the book (594) by the number of chapters (3): 594 / 3 = 198. Therefore, each chapter has 198 pages. The answer is 198.\n"),
+                dict(role='BOT', prompt='To find the number of pages in each chapter, we divide the total number of pages in the book (594) by the number of chapters (3): 594 / 3 = 198. Therefore, each chapter has 198 pages. The answer is 198.\n'),
                 dict(role='HUMAN', prompt="Question: {question}\nLet's think step by step\nAnswer:"),
             ],
         )),

configs/datasets/SuperGLUE_AX_b/SuperGLUE_AX_b_gen_4dfefa.py

@@ -6,8 +6,8 @@ from opencompass.datasets import AXDataset_V2
 from opencompass.utils.text_postprocessors import first_option_postprocess
 
 AX_b_reader_cfg = dict(
-    input_columns=["sentence1", "sentence2"],
-    output_column="label",
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
 )
 
 AX_b_infer_cfg = dict(
@@ -15,9 +15,9 @@ AX_b_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(round=[
             dict(
-                role="HUMAN",
+                role='HUMAN',
                 prompt=
-                "{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:"
+                '{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:'
             ),
         ]),
     ),
@@ -27,15 +27,15 @@ AX_b_infer_cfg = dict(
 
 AX_b_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
 )
 
 AX_b_datasets = [
     dict(
-        abbr="AX_b",
+        abbr='AX_b',
         type=AXDataset_V2,
-        path="./data/SuperGLUE/AX-b/AX-b.jsonl",
+        path='./data/SuperGLUE/AX-b/AX-b.jsonl',
         reader_cfg=AX_b_reader_cfg,
         infer_cfg=AX_b_infer_cfg,
         eval_cfg=AX_b_eval_cfg,

configs/datasets/SuperGLUE_AX_b/SuperGLUE_AX_b_ppl_6db806.py

@@ -5,31 +5,31 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import HFDataset
 
 AX_b_reader_cfg = dict(
-    input_columns=["sentence1", "sentence2"],
-    output_column="label",
-    test_split="train")
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
 
 AX_b_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template={
-            "entailment":
+            'entailment':
             dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    "{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?"
+                    '{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?'
                 ),
-                dict(role="BOT", prompt="Yes"),
+                dict(role='BOT', prompt='Yes'),
             ]),
-            "not_entailment":
+            'not_entailment':
             dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    "{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?"
+                    '{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?'
                 ),
-                dict(role="BOT", prompt="No"),
+                dict(role='BOT', prompt='No'),
             ])
         },
     ),
@@ -42,10 +42,10 @@ AX_b_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 AX_b_datasets = [
     dict(
         type=HFDataset,
-        abbr="AX_b",
-        path="json",
-        data_files="./data/SuperGLUE/AX-b/AX-b.jsonl",
-        split="train",
+        abbr='AX_b',
+        path='json',
+        data_files='./data/SuperGLUE/AX-b/AX-b.jsonl',
+        split='train',
         reader_cfg=AX_b_reader_cfg,
         infer_cfg=AX_b_infer_cfg,
         eval_cfg=AX_b_eval_cfg,

configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen_68aac7.py

@@ -6,8 +6,8 @@ from opencompass.datasets import AXDataset_V2
 from opencompass.utils.text_postprocessors import first_option_postprocess
 
 AX_g_reader_cfg = dict(
-    input_columns=["hypothesis", "premise"],
-    output_column="label",
+    input_columns=['hypothesis', 'premise'],
+    output_column='label',
 )
 
 AX_g_infer_cfg = dict(
@@ -15,9 +15,9 @@ AX_g_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(round=[
             dict(
-                role="HUMAN",
+                role='HUMAN',
                 prompt=
-                "{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:"
+                '{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:'
             ),
         ]),
     ),
@@ -27,15 +27,15 @@ AX_g_infer_cfg = dict(
 
 AX_g_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
 )
 
 AX_g_datasets = [
     dict(
-        abbr="AX_g",
+        abbr='AX_g',
         type=AXDataset_V2,
-        path="./data/SuperGLUE/AX-g/AX-g.jsonl",
+        path='./data/SuperGLUE/AX-g/AX-g.jsonl',
         reader_cfg=AX_g_reader_cfg,
         infer_cfg=AX_g_infer_cfg,
         eval_cfg=AX_g_eval_cfg,

configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_ppl_66caf3.py

@@ -5,31 +5,31 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import HFDataset
 
 AX_g_reader_cfg = dict(
-    input_columns=["hypothesis", "premise"],
-    output_column="label",
-    test_split="train")
+    input_columns=['hypothesis', 'premise'],
+    output_column='label',
+    test_split='train')
 
 AX_g_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template={
-            "entailment":
+            'entailment':
             dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    "{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?"
+                    '{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?'
                 ),
-                dict(role="BOT", prompt="Yes"),
+                dict(role='BOT', prompt='Yes'),
             ]),
-            "not_entailment":
+            'not_entailment':
             dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    "{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?"
+                    '{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?'
                 ),
-                dict(role="BOT", prompt="No"),
+                dict(role='BOT', prompt='No'),
             ])
         },
     ),
@@ -42,10 +42,10 @@ AX_g_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 AX_g_datasets = [
     dict(
         type=HFDataset,
-        abbr="AX_g",
-        path="json",
-        data_files="./data/SuperGLUE/AX-g/AX-g.jsonl",
-        split="train",
+        abbr='AX_g',
+        path='json',
+        data_files='./data/SuperGLUE/AX-g/AX-g.jsonl',
+        split='train',
         reader_cfg=AX_g_reader_cfg,
         infer_cfg=AX_g_infer_cfg,
         eval_cfg=AX_g_eval_cfg,

configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_gen_883d50.py

@@ -6,8 +6,8 @@ from opencompass.datasets import BoolQDataset_V2
 from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 BoolQ_reader_cfg = dict(
-    input_columns=["question", "passage"],
-    output_column="label",
+    input_columns=['question', 'passage'],
+    output_column='label',
 )
 
 BoolQ_infer_cfg = dict(
@@ -15,8 +15,8 @@ BoolQ_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(round=[
             dict(
-                role="HUMAN",
-                prompt="{passage}\nQuestion: {question}\nA. Yes\nB. No\nAnswer:"),
+                role='HUMAN',
+                prompt='{passage}\nQuestion: {question}\nA. Yes\nB. No\nAnswer:'),
         ]),
     ),
     retriever=dict(type=ZeroRetriever),
@@ -25,15 +25,15 @@ BoolQ_infer_cfg = dict(
 
 BoolQ_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 BoolQ_datasets = [
     dict(
-        abbr="BoolQ",
+        abbr='BoolQ',
         type=BoolQDataset_V2,
-        path="./data/SuperGLUE/BoolQ/val.jsonl",
+        path='./data/SuperGLUE/BoolQ/val.jsonl',
         reader_cfg=BoolQ_reader_cfg,
         infer_cfg=BoolQ_infer_cfg,
         eval_cfg=BoolQ_eval_cfg,

configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_314797.py

@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import BoolQDataset_V3
 
 BoolQ_reader_cfg = dict(
-    input_columns=["question", "passage"],
-    output_column="label",
-    test_split="train")
+    input_columns=['question', 'passage'],
+    output_column='label',
+    test_split='train')
 
 BoolQ_infer_cfg = dict(
     prompt_template=dict(
@@ -15,13 +15,13 @@ BoolQ_infer_cfg = dict(
         template={
             'false':
             dict(round=[
-                dict(role="HUMAN", prompt="Passage: {passage}\nQuestion: {question}?"),
-                dict(role="BOT", prompt="Answer: No"),
+                dict(role='HUMAN', prompt='Passage: {passage}\nQuestion: {question}?'),
+                dict(role='BOT', prompt='Answer: No'),
             ]),
             'true':
             dict(round=[
-                dict(role="HUMAN", prompt="Passage: {passage}\nQuestion: {question}?"),
-                dict(role="BOT", prompt="Answer: Yes"),
+                dict(role='HUMAN', prompt='Passage: {passage}\nQuestion: {question}?'),
+                dict(role='BOT', prompt='Answer: Yes'),
             ]),
         },
     ),
@@ -33,9 +33,9 @@ BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 
 BoolQ_datasets = [
     dict(
-        abbr="BoolQ",
+        abbr='BoolQ',
         type=BoolQDataset_V3,
-        path="./data/SuperGLUE/BoolQ/val.jsonl",
+        path='./data/SuperGLUE/BoolQ/val.jsonl',
         reader_cfg=BoolQ_reader_cfg,
         infer_cfg=BoolQ_infer_cfg,
         eval_cfg=BoolQ_eval_cfg,

configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_314b96.py

@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import BoolQDataset
 
 BoolQ_reader_cfg = dict(
-    input_columns=["question", "passage"],
-    output_column="answer",
-    test_split="train")
+    input_columns=['question', 'passage'],
+    output_column='answer',
+    test_split='train')
 
 BoolQ_infer_cfg = dict(
     prompt_template=dict(
@@ -15,13 +15,13 @@ BoolQ_infer_cfg = dict(
         template={
             0:
             dict(round=[
-                dict(role="HUMAN", prompt="{passage}\nQuestion: {question}?"),
-                dict(role="BOT", prompt="No"),
+                dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
+                dict(role='BOT', prompt='No'),
             ]),
             1:
             dict(round=[
-                dict(role="HUMAN", prompt="{passage}\nQuestion: {question}?"),
-                dict(role="BOT", prompt="Yes"),
+                dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
+                dict(role='BOT', prompt='Yes'),
             ]),
         },
     ),
@@ -34,10 +34,10 @@ BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 BoolQ_datasets = [
     dict(
         type=BoolQDataset,
-        abbr="BoolQ",
-        path="json",
-        data_files="./data/SuperGLUE/BoolQ/val.jsonl",
-        split="train",
+        abbr='BoolQ',
+        path='json',
+        data_files='./data/SuperGLUE/BoolQ/val.jsonl',
+        split='train',
         reader_cfg=BoolQ_reader_cfg,
         infer_cfg=BoolQ_infer_cfg,
         eval_cfg=BoolQ_eval_cfg,

configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_4da4db.py

@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import BoolQDataset
 
 BoolQ_reader_cfg = dict(
-    input_columns=["question", "passage"],
-    output_column="answer",
-    test_split="train")
+    input_columns=['question', 'passage'],
+    output_column='answer',
+    test_split='train')
 
 BoolQ_infer_cfg = dict(
     prompt_template=dict(
@@ -15,13 +15,13 @@ BoolQ_infer_cfg = dict(
         template={
             0:
             dict(round=[
-                dict(role="HUMAN", prompt="{passage}\nQuestion: {question}"),
-                dict(role="BOT", prompt="No."),
+                dict(role='HUMAN', prompt='{passage}\nQuestion: {question}'),
+                dict(role='BOT', prompt='No.'),
             ]),
             1:
             dict(round=[
-                dict(role="HUMAN", prompt="{passage}\nQuestion: {question}"),
-                dict(role="BOT", prompt="Yes."),
+                dict(role='HUMAN', prompt='{passage}\nQuestion: {question}'),
+                dict(role='BOT', prompt='Yes.'),
             ]),
         },
     ),
@@ -34,10 +34,10 @@ BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 BoolQ_datasets = [
     dict(
         type=BoolQDataset,
-        abbr="BoolQ",
-        path="json",
-        data_files="./data/SuperGLUE/BoolQ/val.jsonl",
-        split="train",
+        abbr='BoolQ',
+        path='json',
+        data_files='./data/SuperGLUE/BoolQ/val.jsonl',
+        split='train',
         reader_cfg=BoolQ_reader_cfg,
         infer_cfg=BoolQ_infer_cfg,
         eval_cfg=BoolQ_eval_cfg,

configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_9619db.py

@@ -13,8 +13,8 @@ BoolQ_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template={
-            0: "Passage:{passage}。\nQuestion:{question}。\nAnswer: No.",
-            1: "Passage:{passage}。\nQuestion:{question}。\nAnswer: Yes.",
+            0: 'Passage:{passage}。\nQuestion:{question}。\nAnswer: No.',
+            1: 'Passage:{passage}。\nQuestion:{question}。\nAnswer: Yes.',
         }),
     retriever=dict(type=ZeroRetriever),
     inferencer=dict(type=PPLInferencer))

configs/datasets/SuperGLUE_CB/SuperGLUE_CB_gen_854c6c.py

@@ -6,8 +6,8 @@ from opencompass.datasets import CBDataset_V2
 from opencompass.utils.text_postprocessors import first_option_postprocess
 
 CB_reader_cfg = dict(
-    input_columns=["premise", "hypothesis"],
-    output_column="label",
+    input_columns=['premise', 'hypothesis'],
+    output_column='label',
 )
 
 CB_infer_cfg = dict(
@@ -16,9 +16,9 @@ CB_infer_cfg = dict(
         template=dict(
             round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    "{premise}\n{hypothesis}\nWhat is the relation between the two sentences?\nA. Contradiction\nB. Entailment\nC. Neutral\nAnswer:"
+                    '{premise}\n{hypothesis}\nWhat is the relation between the two sentences?\nA. Contradiction\nB. Entailment\nC. Neutral\nAnswer:'
                 ),
             ], ),
     ),
@@ -28,15 +28,15 @@ CB_infer_cfg = dict(
 
 CB_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_option_postprocess, options='ABC'),
 )
 
 CB_datasets = [
     dict(
-        abbr="CB",
+        abbr='CB',
         type=CBDataset_V2,
-        path="./data/SuperGLUE/CB/val.jsonl",
+        path='./data/SuperGLUE/CB/val.jsonl',
         reader_cfg=CB_reader_cfg,
         infer_cfg=CB_infer_cfg,
         eval_cfg=CB_eval_cfg,

configs/datasets/SuperGLUE_CB/SuperGLUE_CB_ppl_0143fe.py

@@ -5,40 +5,40 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import HFDataset
 
 CB_reader_cfg = dict(
-    input_columns=["premise", "hypothesis"],
-    output_column="label",
+    input_columns=['premise', 'hypothesis'],
+    output_column='label',
 )
 
 CB_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template={
-            "contradiction":
+            'contradiction':
             dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    "{premise}\n{hypothesis}\nWhat is the relation between the two sentences?"
+                    '{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
                 ),
-                dict(role="BOT", prompt="Contradiction"),
+                dict(role='BOT', prompt='Contradiction'),
             ]),
-            "entailment":
+            'entailment':
             dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    "{premise}\n{hypothesis}\nWhat is the relation between the two sentences?"
+                    '{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
                 ),
-                dict(role="BOT", prompt="Entailment"),
+                dict(role='BOT', prompt='Entailment'),
             ]),
-            "neutral":
+            'neutral':
             dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    "{premise}\n{hypothesis}\nWhat is the relation between the two sentences?"
+                    '{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
                 ),
-                dict(role="BOT", prompt="Neutral"),
+                dict(role='BOT', prompt='Neutral'),
             ]),
         },
     ),
@@ -51,10 +51,10 @@ CB_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
 CB_datasets = [
     dict(
         type=HFDataset,
-        abbr="CB",
-        path="json",
-        split="train",
-        data_files="./data/SuperGLUE/CB/val.jsonl",
+        abbr='CB',
+        path='json',
+        split='train',
+        data_files='./data/SuperGLUE/CB/val.jsonl',
         reader_cfg=CB_reader_cfg,
         infer_cfg=CB_infer_cfg,
         eval_cfg=CB_eval_cfg,

configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_gen_91ca53.py

@@ -6,8 +6,8 @@ from opencompass.datasets import COPADataset_V2
 from opencompass.utils.text_postprocessors import first_option_postprocess
 
 COPA_reader_cfg = dict(
-    input_columns=["question", "premise", "choice1", "choice2"],
-    output_column="label",
+    input_columns=['question', 'premise', 'choice1', 'choice2'],
+    output_column='label',
 )
 
 COPA_infer_cfg = dict(
@@ -16,9 +16,9 @@ COPA_infer_cfg = dict(
         template=dict(
             round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    "{premise}\nQuestion: Which may be the {question}?\nA. {choice1}\nB. {choice2}\nAnswer:"
+                    '{premise}\nQuestion: Which may be the {question}?\nA. {choice1}\nB. {choice2}\nAnswer:'
                 ),
             ], ),
     ),
@@ -28,15 +28,15 @@ COPA_infer_cfg = dict(
 
 COPA_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
 )
 
 COPA_datasets = [
     dict(
-        abbr="COPA",
+        abbr='COPA',
         type=COPADataset_V2,
-        path="./data/SuperGLUE/COPA/val.jsonl",
+        path='./data/SuperGLUE/COPA/val.jsonl',
         reader_cfg=COPA_reader_cfg,
         infer_cfg=COPA_infer_cfg,
         eval_cfg=COPA_eval_cfg,

configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_54058d.py

@@ -13,8 +13,8 @@ COPA_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template={
-            0: "Premise:{premise}。\nQuestion:{question}。\nAnswer: {choice1}.",
-            1: "Passage:{premise}。\nQuestion:{question}。\nAnswer: {choice2}.",
+            0: 'Premise:{premise}。\nQuestion:{question}。\nAnswer: {choice1}.',
+            1: 'Passage:{premise}。\nQuestion:{question}。\nAnswer: {choice2}.',
         }),
     retriever=dict(type=ZeroRetriever),
     inferencer=dict(type=PPLInferencer))

configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_5c24f1.py

@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import HFDataset
 
 COPA_reader_cfg = dict(
-    input_columns=["question", "premise", "choice1", "choice2"],
-    output_column="label",
-    test_split="train")
+    input_columns=['question', 'premise', 'choice1', 'choice2'],
+    output_column='label',
+    test_split='train')
 
 COPA_infer_cfg = dict(
     prompt_template=dict(
@@ -15,13 +15,13 @@ COPA_infer_cfg = dict(
         template={
             0:
             dict(round=[
-                dict(role="HUMAN", prompt="{premise}\nQuestion: {question}\nAnswer:"),
-                dict(role="BOT", prompt="{choice1}"),
+                dict(role='HUMAN', prompt='{premise}\nQuestion: {question}\nAnswer:'),
+                dict(role='BOT', prompt='{choice1}'),
             ]),
             1:
             dict(round=[
-                dict(role="HUMAN", prompt="{premise}\nQuestion: {question}\nAnswer:"),
-                dict(role="BOT", prompt="{choice2}"),
+                dict(role='HUMAN', prompt='{premise}\nQuestion: {question}\nAnswer:'),
+                dict(role='BOT', prompt='{choice2}'),
             ]),
         },
     ),
@@ -34,10 +34,10 @@ COPA_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 COPA_datasets = [
     dict(
         type=HFDataset,
-        abbr="COPA",
-        path="json",
-        data_files="./data/SuperGLUE/COPA/val.jsonl",
-        split="train",
+        abbr='COPA',
+        path='json',
+        data_files='./data/SuperGLUE/COPA/val.jsonl',
+        split='train',
         reader_cfg=COPA_reader_cfg,
         infer_cfg=COPA_infer_cfg,
         eval_cfg=COPA_eval_cfg,