[Format] Add config lints (#892)

configs/datasets/FewCLUE_chid/FewCLUE_chid_ppl_acccb5.py

@@ -10,7 +10,7 @@ chid_reader_cfg = dict(
 chid_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
-        template={i: f"以下句子是否通顺？\n{{content{i}}}\n这个句子是通顺的。"
+        template={i: f'以下句子是否通顺？\n{{content{i}}}\n这个句子是通顺的。'
                   for i in range(7)}),
     retriever=dict(type=ZeroRetriever),
     inferencer=dict(type=PPLInferencer))

configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_gen_c68933.py

@@ -6,8 +6,8 @@ from opencompass.datasets import CluewscDataset_V2
 from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 cluewsc_reader_cfg = dict(
-    input_columns=["span1", "span2", "text", "new_text"],
-    output_column="label",
+    input_columns=['span1', 'span2', 'text', 'new_text'],
+    output_column='label',
 )
 
 cluewsc_infer_cfg = dict(
@@ -15,9 +15,9 @@ cluewsc_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(round=[
             dict(
-                role="HUMAN",
+                role='HUMAN',
                 prompt=
-                "{text}\n此处，“{span2}”是否指代“{span1}“？\nA. 是\nB. 否\n请从”A“，”B“中进行选择。\n答：",
+                '{text}\n此处，“{span2}”是否指代“{span1}“？\nA. 是\nB. 否\n请从”A“，”B“中进行选择。\n答：',
             ),
         ]),
     ),
@@ -27,23 +27,23 @@ cluewsc_infer_cfg = dict(
 
 cluewsc_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 cluewsc_datasets = [
     dict(
-        abbr="cluewsc-dev",
+        abbr='cluewsc-dev',
         type=CluewscDataset_V2,
-        path="./data/FewCLUE/cluewsc/dev_few_all.json",
+        path='./data/FewCLUE/cluewsc/dev_few_all.json',
         reader_cfg=cluewsc_reader_cfg,
         infer_cfg=cluewsc_infer_cfg,
         eval_cfg=cluewsc_eval_cfg,
     ),
     dict(
-        abbr="cluewsc-test",
+        abbr='cluewsc-test',
         type=CluewscDataset_V2,
-        path="./data/FewCLUE/cluewsc/test_public.json",
+        path='./data/FewCLUE/cluewsc/test_public.json',
         reader_cfg=cluewsc_reader_cfg,
         infer_cfg=cluewsc_infer_cfg,
         eval_cfg=cluewsc_eval_cfg,

configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_12e4e0.py

@@ -15,20 +15,20 @@ cluewsc_infer_cfg = dict(
             0:
             dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
                     "{text}\nHere, is the pronoun \"{span2}\" used to mean \"{span1}\"?"
                 ),
-                dict(role="BOT", prompt="No.")
+                dict(role='BOT', prompt='No.')
             ]),
             1:
             dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
                     "{text}\nHere, is the pronoun \"{span2}\" used to mean \"{span1}\"?"
                 ),
-                dict(role="BOT", prompt="Yes.")
+                dict(role='BOT', prompt='Yes.')
             ]),
         }),
     retriever=dict(type=ZeroRetriever),

configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_868415.py

@@ -15,16 +15,16 @@ cluewsc_infer_cfg = dict(
             0:
             dict(round=[
                 dict(
-                    role="HUMAN",
-                    prompt="{text}\n此处，代词“{span2}“被用于指代“{span1}“吗?"),
-                dict(role="BOT", prompt="否")
+                    role='HUMAN',
+                    prompt='{text}\n此处，代词“{span2}“被用于指代“{span1}“吗?'),
+                dict(role='BOT', prompt='否')
             ]),
             1:
             dict(round=[
                 dict(
-                    role="HUMAN",
-                    prompt="{text}\n此处，代词“{span2}“被用于指代“{span1}“吗?"),
-                dict(role="BOT", prompt="是")
+                    role='HUMAN',
+                    prompt='{text}\n此处，代词“{span2}“被用于指代“{span1}“吗?'),
+                dict(role='BOT', prompt='是')
             ]),
         }),
     retriever=dict(type=ZeroRetriever),

configs/datasets/FewCLUE_csl/FewCLUE_csl_gen_28b223.py

@@ -6,8 +6,8 @@ from opencompass.datasets import CslDataset_V2
 from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 csl_reader_cfg = dict(
-    input_columns=["abst", "keywords"],
-    output_column="label",
+    input_columns=['abst', 'keywords'],
+    output_column='label',
 )
 
 csl_infer_cfg = dict(
@@ -15,9 +15,9 @@ csl_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(round=[
             dict(
-                role="HUMAN",
+                role='HUMAN',
                 prompt=
-                "摘要是对论文内容不加注释和评论的简短陈述，要求扼要地说明研究工作的目的、研究方法和最终结论等。\n关键词是一篇学术论文的核心词汇，一般由一系列名词组成。关键词在全文中应有较高出现频率，且能起到帮助文献检索的作用。\n摘要：{abst}\n关键词：{keywords}\n请问上述关键词是否匹配摘要且符合要求？\nA. 否\nB. 是\n请从”A“，”B“中进行选择。\n答："
+                '摘要是对论文内容不加注释和评论的简短陈述，要求扼要地说明研究工作的目的、研究方法和最终结论等。\n关键词是一篇学术论文的核心词汇，一般由一系列名词组成。关键词在全文中应有较高出现频率，且能起到帮助文献检索的作用。\n摘要：{abst}\n关键词：{keywords}\n请问上述关键词是否匹配摘要且符合要求？\nA. 否\nB. 是\n请从”A“，”B“中进行选择。\n答：'
             )
         ]),
     ),
@@ -27,23 +27,23 @@ csl_infer_cfg = dict(
 
 csl_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 csl_datasets = [
     dict(
-        abbr="csl_dev",
+        abbr='csl_dev',
         type=CslDataset_V2,
-        path="./data/FewCLUE/csl/dev_few_all.json",
+        path='./data/FewCLUE/csl/dev_few_all.json',
         reader_cfg=csl_reader_cfg,
         infer_cfg=csl_infer_cfg,
         eval_cfg=csl_eval_cfg,
     ),
     dict(
-        abbr="csl_test",
+        abbr='csl_test',
         type=CslDataset_V2,
-        path="./data/FewCLUE/csl/test_public.json",
+        path='./data/FewCLUE/csl/test_public.json',
         reader_cfg=csl_reader_cfg,
         infer_cfg=csl_infer_cfg,
         eval_cfg=csl_eval_cfg,

configs/datasets/FewCLUE_csl/FewCLUE_csl_gen_87f4a8.py

@@ -6,8 +6,8 @@ from opencompass.datasets import CslDataset_V2
 from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 csl_reader_cfg = dict(
-    input_columns=["abst", "keywords"],
-    output_column="label",
+    input_columns=['abst', 'keywords'],
+    output_column='label',
 )
 
 csl_infer_cfg = dict(
@@ -15,9 +15,9 @@ csl_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(round=[
             dict(
-                role="HUMAN",
+                role='HUMAN',
                 prompt=
-                "摘要：{abst}\n关键词：{keywords}\n上述关键词出现在学术期刊中是否恰当？\nA. 否\nB. 是\n请从”A“，”B“中进行选择。\n答："
+                '摘要：{abst}\n关键词：{keywords}\n上述关键词出现在学术期刊中是否恰当？\nA. 否\nB. 是\n请从”A“，”B“中进行选择。\n答：'
             )
         ]),
     ),
@@ -27,23 +27,23 @@ csl_infer_cfg = dict(
 
 csl_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 csl_datasets = [
     dict(
-        abbr="csl_dev",
+        abbr='csl_dev',
         type=CslDataset_V2,
-        path="./data/FewCLUE/csl/dev_few_all.json",
+        path='./data/FewCLUE/csl/dev_few_all.json',
         reader_cfg=csl_reader_cfg,
         infer_cfg=csl_infer_cfg,
         eval_cfg=csl_eval_cfg,
     ),
     dict(
-        abbr="csl_test",
+        abbr='csl_test',
         type=CslDataset_V2,
-        path="./data/FewCLUE/csl/test_public.json",
+        path='./data/FewCLUE/csl/test_public.json',
         reader_cfg=csl_reader_cfg,
         infer_cfg=csl_infer_cfg,
         eval_cfg=csl_eval_cfg,

configs/datasets/FewCLUE_csl/FewCLUE_csl_ppl_769f8d.py

@@ -5,17 +5,17 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import CslDataset
 
 csl_reader_cfg = dict(
-    input_columns=["abst", "keywords"], output_column='label')
+    input_columns=['abst', 'keywords'], output_column='label')
 
 csl_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template={
             0:
-            dict(round=[dict(role="HUMAN", prompt="摘要：{abst}")]),
+            dict(round=[dict(role='HUMAN', prompt='摘要：{abst}')]),
             1:
             dict(
-                round=[dict(role="HUMAN", prompt="摘要：{abst}\n关键词：{keywords}")
+                round=[dict(role='HUMAN', prompt='摘要：{abst}\n关键词：{keywords}')
                        ]),
         }),
     retriever=dict(type=ZeroRetriever),

configs/datasets/FewCLUE_csl/FewCLUE_csl_ppl_841b62.py

@@ -5,14 +5,14 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import CslDataset
 
 csl_reader_cfg = dict(
-    input_columns=["abst", "keywords"], output_column='label')
+    input_columns=['abst', 'keywords'], output_column='label')
 
 csl_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template={
-            0: "摘要：{abst}",
-            1: "摘要：{abst}\n关键词：{keywords}"
+            0: '摘要：{abst}',
+            1: '摘要：{abst}\n关键词：{keywords}'
         }),
     retriever=dict(type=ZeroRetriever),
     inferencer=dict(type=PPLInferencer))

configs/datasets/FewCLUE_eprstmt/FewCLUE_eprstmt_gen_740ea0.py

@@ -6,14 +6,14 @@ from opencompass.datasets import eprstmtDataset_V2
 from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 eprstmt_reader_cfg = dict(
-    input_columns=["sentence"], output_column="label", test_split="train")
+    input_columns=['sentence'], output_column='label', test_split='train')
 
 eprstmt_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template=dict(round=[
             dict(
-                role="HUMAN",
+                role='HUMAN',
                 prompt=
                 '内容： "{sentence}"。请对上述内容进行情绪分类。\nA. 积极\nB. 消极\n请从”A“，”B“中进行选择。\n答：'
             ),
@@ -25,23 +25,23 @@ eprstmt_infer_cfg = dict(
 
 eprstmt_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 eprstmt_datasets = [
     dict(
-        abbr="eprstmt-dev",
+        abbr='eprstmt-dev',
         type=eprstmtDataset_V2,
-        path="./data/FewCLUE/eprstmt/dev_few_all.json",
+        path='./data/FewCLUE/eprstmt/dev_few_all.json',
         reader_cfg=eprstmt_reader_cfg,
         infer_cfg=eprstmt_infer_cfg,
         eval_cfg=eprstmt_eval_cfg,
     ),
     dict(
-        abbr="eprstmt-test",
+        abbr='eprstmt-test',
         type=eprstmtDataset_V2,
-        path="./data/FewCLUE/eprstmt/test_public.json",
+        path='./data/FewCLUE/eprstmt/test_public.json',
         reader_cfg=eprstmt_reader_cfg,
         infer_cfg=eprstmt_infer_cfg,
         eval_cfg=eprstmt_eval_cfg,

configs/datasets/FewCLUE_ocnli_fc/FewCLUE_ocnli_fc_gen_f97a97.py

@@ -6,18 +6,18 @@ from opencompass.datasets import cmnliDataset_V2
 from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 ocnli_fc_reader_cfg = dict(
-    input_columns=["sentence1", "sentence2"],
-    output_column="label",
-    test_split="train")
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
 
 ocnli_fc_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template=dict(round=[
             dict(
-                role="HUMAN",
+                role='HUMAN',
                 prompt=
-                "阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}\nA. 对\nB. 错\nC. 可能\n请从“A”，“B”，“C”中进行选择。\n答："
+                '阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}\nA. 对\nB. 错\nC. 可能\n请从“A”，“B”，“C”中进行选择。\n答：'
             ),
         ]),
     ),
@@ -26,23 +26,23 @@ ocnli_fc_infer_cfg = dict(
 )
 ocnli_fc_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 ocnli_fc_datasets = [
     dict(
-        abbr="ocnli_fc-dev",
+        abbr='ocnli_fc-dev',
         type=cmnliDataset_V2,  # ocnli_fc share the same format with cmnli
-        path="./data/FewCLUE/ocnli/dev_few_all.json",
+        path='./data/FewCLUE/ocnli/dev_few_all.json',
         reader_cfg=ocnli_fc_reader_cfg,
         infer_cfg=ocnli_fc_infer_cfg,
         eval_cfg=ocnli_fc_eval_cfg,
     ),
     dict(
-        abbr="ocnli_fc-test",
+        abbr='ocnli_fc-test',
         type=cmnliDataset_V2,  # ocnli_fc share the same format with cmnli
-        path="./data/FewCLUE/ocnli/test_public.json",
+        path='./data/FewCLUE/ocnli/test_public.json',
         reader_cfg=ocnli_fc_reader_cfg,
         infer_cfg=ocnli_fc_infer_cfg,
         eval_cfg=ocnli_fc_eval_cfg,

configs/datasets/FewCLUE_ocnli_fc/FewCLUE_ocnli_fc_ppl_9e8b3d.py

@@ -16,22 +16,22 @@ ocnli_fc_infer_cfg = dict(
             'contradiction':
             dict(round=[
                 dict(
-                    role="HUMAN",
-                    prompt="阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？"),
-                dict(role="BOT", prompt="错")
+                    role='HUMAN',
+                    prompt='阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？'),
+                dict(role='BOT', prompt='错')
             ]),
             'entailment':
             dict(round=[
                 dict(
-                    role="HUMAN",
-                    prompt="阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？"),
-                dict(role="BOT", prompt="对")
+                    role='HUMAN',
+                    prompt='阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？'),
+                dict(role='BOT', prompt='对')
             ]),
             'neutral':
             dict(round=[
                 dict(
-                    role="HUMAN", prompt="如果{sentence1}为真，那么{sentence2}也为真吗？"),
-                dict(role="BOT", prompt="可能")
+                    role='HUMAN', prompt='如果{sentence1}为真，那么{sentence2}也为真吗？'),
+                dict(role='BOT', prompt='可能')
             ]),
         }),
     retriever=dict(type=ZeroRetriever),

configs/datasets/FewCLUE_tnews/FewCLUE_tnews_gen_b90e4a.py

@@ -6,30 +6,30 @@ from opencompass.datasets import TNewsDataset_V2
 from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 tnews_reader_cfg = dict(
-    input_columns="sentence",
-    output_column="label_desc2",
+    input_columns='sentence',
+    output_column='label_desc2',
 )
 
 tnews_labels = [
-    "农业新闻",  # news_agriculture
-    "旅游新闻",  # news_travel
-    "游戏新闻",  # news_game
-    "科技类别公司新闻",  # news_tech
-    "体育类别新闻",  # news_sports
-    "初升高教育新闻",  # news_edu
-    "娱乐圈新闻",  # news_entertainment
-    "投资资讯",  # news_finance
-    "军事类别常识",  # news_military
-    "车辆新闻",  # news_car
-    "楼市新闻",  # news_house
-    "环球不含中国类别新闻",  # news_world
-    "书籍文化历史类别新闻",  # news_culture
-    "故事类别新闻",  # news_story
-    "股票市场类别新闻",  # news_stock
+    '农业新闻',  # news_agriculture
+    '旅游新闻',  # news_travel
+    '游戏新闻',  # news_game
+    '科技类别公司新闻',  # news_tech
+    '体育类别新闻',  # news_sports
+    '初升高教育新闻',  # news_edu
+    '娱乐圈新闻',  # news_entertainment
+    '投资资讯',  # news_finance
+    '军事类别常识',  # news_military
+    '车辆新闻',  # news_car
+    '楼市新闻',  # news_house
+    '环球不含中国类别新闻',  # news_world
+    '书籍文化历史类别新闻',  # news_culture
+    '故事类别新闻',  # news_story
+    '股票市场类别新闻',  # news_stock
 ]
-_tnews_options_list_str = "\n".join(f'{chr(ord("A") + i)}. {tnews_labels[i]}'
+_tnews_options_list_str = '\n'.join(f'{chr(ord("A") + i)}. {tnews_labels[i]}'
                                     for i in range(len(tnews_labels)))
-_tnews_options_range_str = "，".join(f'“{chr(ord("A") + i)}”'
+_tnews_options_range_str = '，'.join(f'“{chr(ord("A") + i)}”'
                                     for i in range(len(tnews_labels)))
 
 tnews_infer_cfg = dict(
@@ -37,9 +37,9 @@ tnews_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(round=[
             dict(
-                role="HUMAN",
+                role='HUMAN',
                 prompt=
-                f"{{sentence}}\n请判断上述内容属于什么新闻？\n{_tnews_options_list_str}\n请从{_tnews_options_range_str}中进行选择。\n答：",
+                f'{{sentence}}\n请判断上述内容属于什么新闻？\n{_tnews_options_list_str}\n请从{_tnews_options_range_str}中进行选择。\n答：',
             ),
         ]),
     ),
@@ -49,23 +49,23 @@ tnews_infer_cfg = dict(
 
 tnews_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_capital_postprocess),
 )
 
 tnews_datasets = [
     dict(
-        abbr="tnews-dev",
+        abbr='tnews-dev',
         type=TNewsDataset_V2,
-        path="./data/FewCLUE/tnews/dev_few_all.json",
+        path='./data/FewCLUE/tnews/dev_few_all.json',
         reader_cfg=tnews_reader_cfg,
         infer_cfg=tnews_infer_cfg,
         eval_cfg=tnews_eval_cfg,
     ),
     dict(
-        abbr="tnews-test",
+        abbr='tnews-test',
         type=TNewsDataset_V2,
-        path="./data/FewCLUE/tnews/test_public.json",
+        path='./data/FewCLUE/tnews/test_public.json',
         reader_cfg=tnews_reader_cfg,
         infer_cfg=tnews_infer_cfg,
         eval_cfg=tnews_eval_cfg,

configs/datasets/FinanceIQ/FinanceIQ_gen_e0e6b5.py

@@ -6,29 +6,29 @@ from opencompass.datasets import FinanceIQDataset
 from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 financeIQ_subject_mapping_en = {
-    'certified_public_accountant': '注册会计师（CPA）', 
+    'certified_public_accountant': '注册会计师（CPA）',
     'banking_qualification': '银行从业资格',
-    'securities_qualification': '证券从业资格', 
-    'fund_qualification': '基金从业资格', 
-    'insurance_qualification': '保险从业资格CICE', 
-    'economic_analyst': '经济师', 
-    'taxation_practitioner': '税务师', 
-    'futures_qualification': '期货从业资格', 
+    'securities_qualification': '证券从业资格',
+    'fund_qualification': '基金从业资格',
+    'insurance_qualification': '保险从业资格CICE',
+    'economic_analyst': '经济师',
+    'taxation_practitioner': '税务师',
+    'futures_qualification': '期货从业资格',
     'certified_fin_planner': '理财规划师',
-    'actuary_fin_math': '精算师-金融数学', 
+    'actuary_fin_math': '精算师-金融数学',
 }
 
 financeIQ_subject_mapping = {
-    '注册会计师（CPA）': '注册会计师（CPA）', 
+    '注册会计师（CPA）': '注册会计师（CPA）',
     '银行从业资格': '银行从业资格',
-    '证券从业资格': '证券从业资格', 
-    '基金从业资格': '基金从业资格', 
-    '保险从业资格CICE': '保险从业资格CICE', 
-    '经济师': '经济师', 
-    '税务师': '税务师', 
-    '期货从业资格': '期货从业资格', 
+    '证券从业资格': '证券从业资格',
+    '基金从业资格': '基金从业资格',
+    '保险从业资格CICE': '保险从业资格CICE',
+    '经济师': '经济师',
+    '税务师': '税务师',
+    '期货从业资格': '期货从业资格',
     '理财规划师': '理财规划师',
-    '精算师-金融数学': '精算师-金融数学', 
+    '精算师-金融数学': '精算师-金融数学',
 }
 
 financeIQ_all_sets = list(financeIQ_subject_mapping.keys())
@@ -40,16 +40,16 @@ for _name in financeIQ_all_sets:
         ice_template=dict(
             type=PromptTemplate,
             template=dict(
-                begin="</E>",
+                begin='</E>',
                 round=[
                     dict(
-                        role="HUMAN",
+                        role='HUMAN',
                         prompt=
-                        f"以下是关于{_ch_name}的单项选择题，请直接给出正确答案的选项。\n题目：{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}"
+                        f'以下是关于{_ch_name}的单项选择题，请直接给出正确答案的选项。\n题目：{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}'
                     ),
-                    dict(role="BOT", prompt='答案是: {answer}'),
+                    dict(role='BOT', prompt='答案是: {answer}'),
                 ]),
-            ice_token="</E>",
+            ice_token='</E>',
         ),
         retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
         inferencer=dict(type=GenInferencer),
@@ -62,13 +62,13 @@ for _name in financeIQ_all_sets:
     financeIQ_datasets.append(
         dict(
             type=FinanceIQDataset,
-            path="./data/FinanceIQ/",
+            path='./data/FinanceIQ/',
             name=_name,
-            abbr=f"FinanceIQ-{_name}",
+            abbr=f'FinanceIQ-{_name}',
             reader_cfg=dict(
-                input_columns=["question", "A", "B", "C", "D"],
-                output_column="answer",
-                train_split="dev",
+                input_columns=['question', 'A', 'B', 'C', 'D'],
+                output_column='answer',
+                train_split='dev',
                 test_split='test'),
             infer_cfg=financeIQ_infer_cfg,
             eval_cfg=financeIQ_eval_cfg,

configs/datasets/FinanceIQ/FinanceIQ_ppl_42b9bd.py

@@ -5,29 +5,29 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import FinanceIQDataset
 
 financeIQ_subject_mapping_en = {
-    'certified_public_accountant': '注册会计师（CPA）', 
+    'certified_public_accountant': '注册会计师（CPA）',
     'banking_qualification': '银行从业资格',
-    'securities_qualification': '证券从业资格', 
-    'fund_qualification': '基金从业资格', 
-    'insurance_qualification': '保险从业资格CICE', 
-    'economic_analyst': '经济师', 
-    'taxation_practitioner': '税务师', 
-    'futures_qualification': '期货从业资格', 
+    'securities_qualification': '证券从业资格',
+    'fund_qualification': '基金从业资格',
+    'insurance_qualification': '保险从业资格CICE',
+    'economic_analyst': '经济师',
+    'taxation_practitioner': '税务师',
+    'futures_qualification': '期货从业资格',
     'certified_fin_planner': '理财规划师',
-    'actuary_fin_math': '精算师-金融数学', 
+    'actuary_fin_math': '精算师-金融数学',
 }
 
 financeIQ_subject_mapping = {
-    '注册会计师（CPA）': '注册会计师（CPA）', 
+    '注册会计师（CPA）': '注册会计师（CPA）',
     '银行从业资格': '银行从业资格',
-    '证券从业资格': '证券从业资格', 
-    '基金从业资格': '基金从业资格', 
-    '保险从业资格CICE': '保险从业资格CICE', 
-    '经济师': '经济师', 
-    '税务师': '税务师', 
-    '期货从业资格': '期货从业资格', 
+    '证券从业资格': '证券从业资格',
+    '基金从业资格': '基金从业资格',
+    '保险从业资格CICE': '保险从业资格CICE',
+    '经济师': '经济师',
+    '税务师': '税务师',
+    '期货从业资格': '期货从业资格',
     '理财规划师': '理财规划师',
-    '精算师-金融数学': '精算师-金融数学', 
+    '精算师-金融数学': '精算师-金融数学',
 }
 
 financeIQ_all_sets = list(financeIQ_subject_mapping.keys())
@@ -40,17 +40,17 @@ for _name in financeIQ_all_sets:
             type=PromptTemplate,
             template={
                 answer: dict(
-                    begin="</E>",
+                    begin='</E>',
                     round=[
                         dict(
-                            role="HUMAN",
-                            prompt=f"以下是关于{_ch_name}的单项选择题，请直接给出正确答案的选项。\n题目：{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}"
+                            role='HUMAN',
+                            prompt=f'以下是关于{_ch_name}的单项选择题，请直接给出正确答案的选项。\n题目：{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}'
                         ),
-                        dict(role="BOT", prompt=f'答案是: {answer}'),
+                        dict(role='BOT', prompt=f'答案是: {answer}'),
                     ])
-                for answer in ["A", "B", "C", "D"]
+                for answer in ['A', 'B', 'C', 'D']
             },
-            ice_token="</E>",
+            ice_token='</E>',
         ),
         retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
         inferencer=dict(type=PPLInferencer),
@@ -61,13 +61,13 @@ for _name in financeIQ_all_sets:
     financeIQ_datasets.append(
         dict(
             type=FinanceIQDataset,
-            path="./data/FinanceIQ/",
+            path='./data/FinanceIQ/',
             name=_name,
-            abbr=f"FinanceIQ-{_name}",
+            abbr=f'FinanceIQ-{_name}',
             reader_cfg=dict(
-                input_columns=["question", "A", "B", "C", "D"],
-                output_column="answer",
-                train_split="dev",
+                input_columns=['question', 'A', 'B', 'C', 'D'],
+                output_column='answer',
+                train_split='dev',
                 test_split='test'),
             infer_cfg=financeIQ_infer_cfg,
             eval_cfg=financeIQ_eval_cfg,

configs/datasets/GLUE_CoLA/GLUE_CoLA_ppl_77d0df.py

@@ -5,20 +5,20 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import HFDataset
 
 
-_hint = "The following are text classification questions. \n" \
-    "Please determine whether the following sentence is linguistically acceptable: " \
-    "0 means unacceptable, 1 means acceptable.\n"
+_hint = 'The following are text classification questions. \n' \
+    'Please determine whether the following sentence is linguistically acceptable: ' \
+    '0 means unacceptable, 1 means acceptable.\n'
 
 CoLA_infer_cfg = dict(
     ice_template=dict(
         type=PromptTemplate,
-        template="Sentence: {sentence}\nResult: {label}",
+        template='Sentence: {sentence}\nResult: {label}',
     ),
     prompt_template=dict(
         type=PromptTemplate,
         template={
             answer:
-            f"{_hint}</E>Sentence: {{sentence}}\nResult: {answer}"
+            f'{_hint}</E>Sentence: {{sentence}}\nResult: {answer}'
             for answer in [0, 1]
         },
         ice_token='</E>',
@@ -29,7 +29,7 @@ CoLA_infer_cfg = dict(
 CoLA_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
 
 CoLA_datasets = []
-for _split in ["validation"]:
+for _split in ['validation']:
 
     CoLA_reader_cfg = dict(
         input_columns=['sentence'],

configs/datasets/GLUE_MRPC/GLUE_MRPC_ppl_96564c.py

@@ -5,19 +5,19 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import HFDataset
 
 
-_hint = "The following are semantic matching questions. \n" \
-    "Please determine whether the following two sentences are semantically equivalent: " \
-    "0 means not equivalent, 1 means equivalent.\n"
+_hint = 'The following are semantic matching questions. \n' \
+    'Please determine whether the following two sentences are semantically equivalent: ' \
+    '0 means not equivalent, 1 means equivalent.\n'
 MRPC_infer_cfg = dict(
     ice_template=dict(
         type=PromptTemplate,
-        template="Sentence one: {sentence1}\nSentence two: {sentence2}\nResult: {label}",
+        template='Sentence one: {sentence1}\nSentence two: {sentence2}\nResult: {label}',
     ),
     prompt_template=dict(
         type=PromptTemplate,
         template={
             answer:
-            f"{_hint}</E>Sentence one: {{sentence1}}\nSentence two: {{sentence2}}\nResult: {answer}"
+            f'{_hint}</E>Sentence one: {{sentence1}}\nSentence two: {{sentence2}}\nResult: {answer}'
             for answer in [0, 1]
         },
         ice_token='</E>',
@@ -29,12 +29,12 @@ MRPC_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
 
 
 MRPC_datasets = []
-for _split in ["validation", "test"]:
+for _split in ['validation', 'test']:
 
     MRPC_reader_cfg = dict(
         input_columns=['sentence1', 'sentence2'],
         output_column='label',
-        train_split="train",
+        train_split='train',
         test_split=_split
     )
 

configs/datasets/GLUE_QQP/GLUE_QQP_ppl_250d00.py

@@ -5,19 +5,19 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import HFDataset
 
 
-_hint = "The following are semantic matching questions. \n" \
-    "Please determine whether the following two sentences are semantically duplicate: " \
-    "0 means not duplicate, 1 means duplicate.\n"
+_hint = 'The following are semantic matching questions. \n' \
+    'Please determine whether the following two sentences are semantically duplicate: ' \
+    '0 means not duplicate, 1 means duplicate.\n'
 QQP_infer_cfg = dict(
     ice_template=dict(
         type=PromptTemplate,
-        template="Sentence one: {question1}\nSentence two: {question2}\nResult: {label}",
+        template='Sentence one: {question1}\nSentence two: {question2}\nResult: {label}',
     ),
     prompt_template=dict(
         type=PromptTemplate,
         template={
             answer:
-            f"{_hint}</E>Sentence one: {{question1}}\nSentence two: {{question2}}\nResult: {answer}"
+            f'{_hint}</E>Sentence one: {{question1}}\nSentence two: {{question2}}\nResult: {answer}'
             for answer in [0, 1]
         },
         ice_token='</E>',
@@ -29,12 +29,12 @@ QQP_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
 
 
 QQP_datasets = []
-for _split in ["validation", "test"]:
+for _split in ['validation', 'test']:
 
     QQP_reader_cfg = dict(
         input_columns=['question1', 'question2'],
         output_column='label',
-        train_split="train",
+        train_split='train',
         test_split=_split
     )
 

configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_4c31db.py

@@ -10,33 +10,33 @@ with read_base():
 
 GaokaoBench_datasets = []
 for folder, prompts in [
-    ("Multiple-choice_Questions", MCQ_prompts),
-    ("Fill-in-the-blank_Questions", FBQ_prompts),
+    ('Multiple-choice_Questions', MCQ_prompts),
+    ('Fill-in-the-blank_Questions', FBQ_prompts),
 ]:
     for p in prompts:
         reader_cfg = {
-            "input_columns": ["question"],
-            "output_column": "answer",
+            'input_columns': ['question'],
+            'output_column': 'answer',
         }
         infer_cfg = {
-            "ice_template": {
-                "type": PromptTemplate,
-                "template": {"round": [{"role": "HUMAN", "prompt": p["prefix_prompt"] + "{question}"}]},
-                "ice_token": "</E>",
+            'ice_template': {
+                'type': PromptTemplate,
+                'template': {'round': [{'role': 'HUMAN', 'prompt': p['prefix_prompt'] + '{question}'}]},
+                'ice_token': '</E>',
             },
-            "retriever": {"type": ZeroRetriever},
-            "inferencer": {"type": GenInferencer, "max_out_len": 1024},
+            'retriever': {'type': ZeroRetriever},
+            'inferencer': {'type': GenInferencer, 'max_out_len': 1024},
         }
         eval_cfg = {
-            "evaluator": {"type": "GaokaoBenchEvaluator" + "_" + p["type"]},
-            "pred_role": "BOT",
+            'evaluator': {'type': 'GaokaoBenchEvaluator' + '_' + p['type']},
+            'pred_role': 'BOT',
         }
         dataset = {
-            "type": GaokaoBenchDataset,
-            "abbr": "GaokaoBench_" + p["keyword"],
-            "path": os.path.join("data", "GAOKAO-BENCH", "data", folder, p["keyword"] + ".json"),
-            "reader_cfg": reader_cfg,
-            "infer_cfg": infer_cfg,
-            "eval_cfg": eval_cfg,
+            'type': GaokaoBenchDataset,
+            'abbr': 'GaokaoBench_' + p['keyword'],
+            'path': os.path.join('data', 'GAOKAO-BENCH', 'data', folder, p['keyword'] + '.json'),
+            'reader_cfg': reader_cfg,
+            'infer_cfg': infer_cfg,
+            'eval_cfg': eval_cfg,
         }
         GaokaoBench_datasets.append(dataset)