[Format] Add config lints (#892)

configs/datasets/anthropics_evals/sycophancy_gen_4bba45.py

@@ -24,7 +24,7 @@ sycophancy_infer_cfg = dict(
 
 sycophancy_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_option_postprocess, options='ABCDEFG'),
 )
 

configs/datasets/apps/apps_gen_c7893a.py

@@ -3,23 +3,23 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.datasets import APPSDataset, APPSEvaluator
 
-APPS_reader_cfg = dict(input_columns=["question", "starter"], output_column="problem_id", train_split='test')
+APPS_reader_cfg = dict(input_columns=['question', 'starter'], output_column='problem_id', train_split='test')
 
 APPS_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
-        template="Please write a python program to address the following QUESTION. Your ANSWER should be in a code block format like this: ```python # Write your code here ```. \nQUESTION:\n{question} {starter}\nANSWER:\n"),
+        template='Please write a python program to address the following QUESTION. Your ANSWER should be in a code block format like this: ```python # Write your code here ```. \nQUESTION:\n{question} {starter}\nANSWER:\n'),
     retriever=dict(type=ZeroRetriever),
     inferencer=dict(type=GenInferencer, max_out_len=512),
 )
 
-APPS_eval_cfg = dict(evaluator=dict(type=APPSEvaluator), pred_role="BOT")
+APPS_eval_cfg = dict(evaluator=dict(type=APPSEvaluator), pred_role='BOT')
 
 APPS_datasets = [
     dict(
         type=APPSDataset,
-        abbr="apps",
-        path="codeparrot/apps",
+        abbr='apps',
+        path='codeparrot/apps',
         num_repeats=1,
         reader_cfg=APPS_reader_cfg,
         infer_cfg=APPS_infer_cfg,

configs/datasets/apps/apps_mini_gen_c7893a.py

@@ -3,23 +3,23 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.datasets import APPS_miniDataset, APPSEvaluator
 
-APPS_reader_cfg = dict(input_columns=["question", "starter"], output_column="problem_id", train_split='test')
+APPS_reader_cfg = dict(input_columns=['question', 'starter'], output_column='problem_id', train_split='test')
 
 APPS_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
-        template="Please write a python program to address the following QUESTION. Your ANSWER should be in a code block format like this: ```python # Write your code here ```. \nQUESTION:\n{question} {starter}\nANSWER:\n"),
+        template='Please write a python program to address the following QUESTION. Your ANSWER should be in a code block format like this: ```python # Write your code here ```. \nQUESTION:\n{question} {starter}\nANSWER:\n'),
     retriever=dict(type=ZeroRetriever),
     inferencer=dict(type=GenInferencer, max_out_len=512),
 )
 
-APPS_eval_cfg = dict(evaluator=dict(type=APPSEvaluator), pred_role="BOT")
+APPS_eval_cfg = dict(evaluator=dict(type=APPSEvaluator), pred_role='BOT')
 
 APPS_mini_datasets = [
     dict(
         type=APPS_miniDataset,
-        abbr="apps_mini",
-        path="./data/apps_mini",
+        abbr='apps_mini',
+        path='./data/apps_mini',
         num_repeats=1,
         reader_cfg=APPS_reader_cfg,
         infer_cfg=APPS_infer_cfg,

configs/datasets/bbh/bbh_gen_0a5495.py

@@ -5,7 +5,7 @@ from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import BBHDataset, BBHEvaluator, bbh_mcq_postprocess, BBHEvaluator_mcq
 
-bbh_reader_cfg = dict(input_columns=["input"], output_column="target")
+bbh_reader_cfg = dict(input_columns=['input'], output_column='target')
 
 bbh_multiple_choice_sets = [
     'temporal_sequences',
@@ -52,14 +52,14 @@ for _name in bbh_multiple_choice_sets:
         inferencer=dict(type=GenInferencer, max_out_len=512))
     bbh_eval_cfg = dict(
         evaluator=dict(type=BBHEvaluator_mcq),
-        pred_role="BOT",
+        pred_role='BOT',
         pred_postprocessor=dict(type=bbh_mcq_postprocess),
         dataset_postprocessor=dict(type=bbh_mcq_postprocess))
 
     bbh_datasets.append(
         dict(
             type=BBHDataset,
-            path=f"./data/BBH/data",
+            path=f'./data/BBH/data',
             name=_name,
             abbr='bbh-' + _name,
             reader_cfg=bbh_reader_cfg,
@@ -76,12 +76,12 @@ for _name in bbh_free_form_sets:
         ),
         retriever=dict(type=ZeroRetriever),
         inferencer=dict(type=GenInferencer, max_out_len=512))
-    bbh_eval_cfg = dict(evaluator=dict(type=BBHEvaluator), pred_role="BOT")
+    bbh_eval_cfg = dict(evaluator=dict(type=BBHEvaluator), pred_role='BOT')
 
     bbh_datasets.append(
         dict(
             type=BBHDataset,
-            path=f"./data/BBH/data",
+            path=f'./data/BBH/data',
             name=_name,
             abbr='bbh-' + _name,
             reader_cfg=bbh_reader_cfg,

configs/datasets/bbh/bbh_gen_2879b0.py

@@ -23,11 +23,11 @@ for name, test_type in settings:
             desc = task_prompt.strip() + '\n'
         else:
             desc = ''
-        prompt_rounds.append(dict(role="HUMAN", prompt=f"{desc}{question.strip()}"))
-        prompt_rounds.append(dict(role="BOT", prompt=answer.strip()))
-    prompt_rounds.append(dict(role="HUMAN", prompt="Q: {input}"))
+        prompt_rounds.append(dict(role='HUMAN', prompt=f'{desc}{question.strip()}'))
+        prompt_rounds.append(dict(role='BOT', prompt=answer.strip()))
+    prompt_rounds.append(dict(role='HUMAN', prompt='Q: {input}'))
 
-    bbh_reader_cfg = dict(input_columns=["input"], output_column="target")
+    bbh_reader_cfg = dict(input_columns=['input'], output_column='target')
 
     bbh_infer_cfg = dict(
         prompt_template=dict(type=PromptTemplate, template=dict(round=prompt_rounds)),
@@ -37,18 +37,18 @@ for name, test_type in settings:
     if test_type == 'mcq':
         bbh_eval_cfg = dict(
             evaluator=dict(type=BBHEvaluator_mcq),
-            pred_role="BOT",
+            pred_role='BOT',
             pred_postprocessor=dict(type=bbh_mcq_postprocess),
             dataset_postprocessor=dict(type=bbh_mcq_postprocess))
     else:
         bbh_eval_cfg = dict(
             evaluator=dict(type=BBHEvaluator),
-            pred_role="BOT")
+            pred_role='BOT')
 
     bbh_datasets.append(
         dict(
             type=BBHDataset,
-            path="./data/BBH/data",
+            path='./data/BBH/data',
             name=name,
             abbr='bbh-' + name,
             reader_cfg=bbh_reader_cfg.copy(),

configs/datasets/bbh/bbh_gen_5b92b0.py

@@ -5,7 +5,7 @@ from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import BBHDataset, BBHEvaluator, bbh_mcq_postprocess, BBHEvaluator_mcq
 
-bbh_reader_cfg = dict(input_columns=["input"], output_column="target")
+bbh_reader_cfg = dict(input_columns=['input'], output_column='target')
 
 bbh_multiple_choice_sets = [
     'temporal_sequences',
@@ -48,7 +48,7 @@ for _name in bbh_multiple_choice_sets:
             type=PromptTemplate,
             template=dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
                     f"Follow the given examples and answer the question.\n{_hint}\n\nQ: {{input}}\nA: Let's think step by step."
                 )
@@ -57,14 +57,14 @@ for _name in bbh_multiple_choice_sets:
         inferencer=dict(type=GenInferencer, max_out_len=512))
     bbh_eval_cfg = dict(
         evaluator=dict(type=BBHEvaluator_mcq),
-        pred_role="BOT",
+        pred_role='BOT',
         pred_postprocessor=dict(type=bbh_mcq_postprocess),
         dataset_postprocessor=dict(type=bbh_mcq_postprocess))
 
     bbh_datasets.append(
         dict(
             type=BBHDataset,
-            path=f"./data/BBH/data",
+            path=f'./data/BBH/data',
             name=_name,
             abbr='bbh-' + _name,
             reader_cfg=bbh_reader_cfg,
@@ -79,19 +79,19 @@ for _name in bbh_free_form_sets:
             type=PromptTemplate,
             template=dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
                     f"Follow the given examples and answer the question.\n{_hint}\n\nQ: {{input}}\nA: Let's think step by step."
                 )
             ])),
         retriever=dict(type=ZeroRetriever),
         inferencer=dict(type=GenInferencer, max_out_len=512))
-    bbh_eval_cfg = dict(evaluator=dict(type=BBHEvaluator), pred_role="BOT")
+    bbh_eval_cfg = dict(evaluator=dict(type=BBHEvaluator), pred_role='BOT')
 
     bbh_datasets.append(
         dict(
             type=BBHDataset,
-            path=f"./data/BBH/data",
+            path=f'./data/BBH/data',
             name=_name,
             abbr='bbh-' + _name,
             reader_cfg=bbh_reader_cfg,

configs/datasets/bbh/bbh_gen_5bf00b.py

@@ -5,7 +5,7 @@ from opencompass.openicl.icl_inferencer import GenInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
 from opencompass.datasets import BBHDataset, BBHEvaluator, bbh_mcq_postprocess, BBHEvaluator_mcq
 
-bbh_reader_cfg = dict(input_columns=["input"], output_column="target")
+bbh_reader_cfg = dict(input_columns=['input'], output_column='target')
 
 bbh_multiple_choice_sets = [
     'temporal_sequences',
@@ -48,23 +48,23 @@ for _name in bbh_multiple_choice_sets:
             type=PromptTemplate,
             template=dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    f"Follow the given examples and answer the question.\n{_hint}\n\nQ: {{input}}\nA: "
+                    f'Follow the given examples and answer the question.\n{_hint}\n\nQ: {{input}}\nA: '
                 )
             ])),
         retriever=dict(type=ZeroRetriever),
         inferencer=dict(type=GenInferencer, max_out_len=512))
     bbh_eval_cfg = dict(
         evaluator=dict(type=BBHEvaluator_mcq),
-        pred_role="BOT",
+        pred_role='BOT',
         pred_postprocessor=dict(type=bbh_mcq_postprocess),
         dataset_postprocessor=dict(type=bbh_mcq_postprocess))
 
     bbh_datasets.append(
         dict(
             type=BBHDataset,
-            path=f"./data/BBH/data",
+            path=f'./data/BBH/data',
             name=_name,
             abbr='bbh-' + _name,
             reader_cfg=bbh_reader_cfg,
@@ -79,19 +79,19 @@ for _name in bbh_free_form_sets:
             type=PromptTemplate,
             template=dict(round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    f"Follow the given examples and answer the question.\n{_hint}\n\nQ: {{input}}\nA: "
+                    f'Follow the given examples and answer the question.\n{_hint}\n\nQ: {{input}}\nA: '
                 )
             ])),
         retriever=dict(type=ZeroRetriever),
         inferencer=dict(type=GenInferencer, max_out_len=512))
-    bbh_eval_cfg = dict(evaluator=dict(type=BBHEvaluator), pred_role="BOT")
+    bbh_eval_cfg = dict(evaluator=dict(type=BBHEvaluator), pred_role='BOT')
 
     bbh_datasets.append(
         dict(
             type=BBHDataset,
-            path=f"./data/BBH/data",
+            path=f'./data/BBH/data',
             name=_name,
             abbr='bbh-' + _name,
             reader_cfg=bbh_reader_cfg,

configs/datasets/ceval/ceval_clean_ppl.py

@@ -62,7 +62,7 @@ ceval_subject_mapping = {
 ceval_all_sets = list(ceval_subject_mapping.keys())
 
 ceval_datasets = []
-for _split in ["val"]:
+for _split in ['val']:
     for _name in ceval_all_sets:
         _ch_name = ceval_subject_mapping[_name][1]
         ceval_infer_cfg = dict(
@@ -70,18 +70,18 @@ for _split in ["val"]:
                 type=PromptTemplate,
                 template={
                     answer: dict(
-                        begin="</E>",
+                        begin='</E>',
                         round=[
                             dict(
-                                role="HUMAN",
+                                role='HUMAN',
                                 prompt=
-                                f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: "
+                                f'以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: '
                             ),
-                            dict(role="BOT", prompt=answer),
+                            dict(role='BOT', prompt=answer),
                         ])
-                    for answer in ["A", "B", "C", "D"]
+                    for answer in ['A', 'B', 'C', 'D']
                 },
-                ice_token="</E>",
+                ice_token='</E>',
             ),
             retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
             inferencer=dict(type=PPLInferencer),
@@ -92,13 +92,13 @@ for _split in ["val"]:
         ceval_datasets.append(
             dict(
                 type=CEvalDataset,
-                path="./data/ceval/formal_ceval",
+                path='./data/ceval/formal_ceval',
                 name=_name,
-                abbr="ceval-" + _name if _split == "val" else "ceval-test-" + _name,
+                abbr='ceval-' + _name if _split == 'val' else 'ceval-test-' + _name,
                 reader_cfg=dict(
-                    input_columns=["question", "A", "B", "C", "D"],
-                    output_column="answer",
-                    train_split="dev",
+                    input_columns=['question', 'A', 'B', 'C', 'D'],
+                    output_column='answer',
+                    train_split='dev',
                     test_split=_split),
                 infer_cfg=ceval_infer_cfg,
                 eval_cfg=ceval_eval_cfg,

configs/datasets/ceval/ceval_gen_2daf24.py

@@ -62,23 +62,23 @@ ceval_subject_mapping = {
 ceval_all_sets = list(ceval_subject_mapping.keys())
 
 ceval_datasets = []
-for _split in ["val", "test"]:
+for _split in ['val', 'test']:
     for _name in ceval_all_sets:
         _ch_name = ceval_subject_mapping[_name][1]
         ceval_infer_cfg = dict(
             ice_template=dict(
                 type=PromptTemplate,
                 template=dict(
-                    begin="</E>",
+                    begin='</E>',
                     round=[
                         dict(
-                            role="HUMAN",
+                            role='HUMAN',
                             prompt=
-                            f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: "
+                            f'以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: '
                         ),
-                        dict(role="BOT", prompt="{answer}"),
+                        dict(role='BOT', prompt='{answer}'),
                     ]),
-                ice_token="</E>",
+                ice_token='</E>',
             ),
             retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
             inferencer=dict(type=GenInferencer),
@@ -91,14 +91,14 @@ for _split in ["val", "test"]:
         ceval_datasets.append(
             dict(
                 type=CEvalDataset,
-                path="./data/ceval/formal_ceval",
+                path='./data/ceval/formal_ceval',
                 name=_name,
-                abbr="ceval-" + _name if _split == "val" else "ceval-test-" +
+                abbr='ceval-' + _name if _split == 'val' else 'ceval-test-' +
                 _name,
                 reader_cfg=dict(
-                    input_columns=["question", "A", "B", "C", "D"],
-                    output_column="answer",
-                    train_split="dev",
+                    input_columns=['question', 'A', 'B', 'C', 'D'],
+                    output_column='answer',
+                    train_split='dev',
                     test_split=_split),
                 infer_cfg=ceval_infer_cfg,
                 eval_cfg=ceval_eval_cfg,

configs/datasets/ceval/ceval_gen_5f30c7.py

@@ -62,23 +62,23 @@ ceval_subject_mapping = {
 ceval_all_sets = list(ceval_subject_mapping.keys())
 
 ceval_datasets = []
-for _split in ["val"]:
+for _split in ['val']:
     for _name in ceval_all_sets:
         _ch_name = ceval_subject_mapping[_name][1]
         ceval_infer_cfg = dict(
             ice_template=dict(
                 type=PromptTemplate,
                 template=dict(
-                    begin="</E>",
+                    begin='</E>',
                     round=[
                         dict(
-                            role="HUMAN",
+                            role='HUMAN',
                             prompt=
-                            f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: "
+                            f'以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: '
                         ),
-                        dict(role="BOT", prompt="{answer}"),
+                        dict(role='BOT', prompt='{answer}'),
                     ]),
-                ice_token="</E>",
+                ice_token='</E>',
             ),
             retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
             inferencer=dict(type=GenInferencer),
@@ -91,14 +91,14 @@ for _split in ["val"]:
         ceval_datasets.append(
             dict(
                 type=CEvalDataset,
-                path="./data/ceval/formal_ceval",
+                path='./data/ceval/formal_ceval',
                 name=_name,
-                abbr="ceval-" + _name if _split == "val" else "ceval-test-" +
+                abbr='ceval-' + _name if _split == 'val' else 'ceval-test-' +
                 _name,
                 reader_cfg=dict(
-                    input_columns=["question", "A", "B", "C", "D"],
-                    output_column="answer",
-                    train_split="dev",
+                    input_columns=['question', 'A', 'B', 'C', 'D'],
+                    output_column='answer',
+                    train_split='dev',
                     test_split=_split),
                 infer_cfg=ceval_infer_cfg,
                 eval_cfg=ceval_eval_cfg,

configs/datasets/ceval/ceval_internal_ppl_1cd8bf.py

@@ -61,28 +61,28 @@ ceval_subject_mapping = {
 ceval_all_sets = list(ceval_subject_mapping.keys())
 
 ceval_datasets = []
-for _split in ["val", "test"]:
+for _split in ['val', 'test']:
     for _name in ceval_all_sets:
         ceval_reader_cfg = dict(
-            input_columns=["question", "A", "B", "C", "D"],
-            output_column="answer",
-            train_split="dev",
+            input_columns=['question', 'A', 'B', 'C', 'D'],
+            output_column='answer',
+            train_split='dev',
             test_split=_split,
         )
 
         _ch_name = ceval_subject_mapping[_name][1]
 
-        hint = f"以下是关于{_ch_name}的单项选择题，请直接给出正确答案的选项。"
-        question_and_options = "{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}"
+        hint = f'以下是关于{_ch_name}的单项选择题，请直接给出正确答案的选项。'
+        question_and_options = '{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}'
         ceval_infer_cfg = dict(
             ice_template=dict(
                 type=PromptTemplate,
-                template={answer: f"{question_and_options}\n答案: {answer}\n" for answer in ["A", "B", "C", "D"]},
+                template={answer: f'{question_and_options}\n答案: {answer}\n' for answer in ['A', 'B', 'C', 'D']},
             ),
             prompt_template=dict(
                 type=PromptTemplate,
-                template={answer: f"{hint}\n</E>{question_and_options}\n答案: {answer}" for answer in ["A", "B", "C", "D"]},
-                ice_token="</E>",
+                template={answer: f'{hint}\n</E>{question_and_options}\n答案: {answer}' for answer in ['A', 'B', 'C', 'D']},
+                ice_token='</E>',
             ),
             retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
             inferencer=dict(type=PPLInferencer),
@@ -93,9 +93,9 @@ for _split in ["val", "test"]:
         ceval_datasets.append(
             dict(
                 type=CEvalDataset,
-                path="./data/ceval_internal/formal_ceval",
+                path='./data/ceval_internal/formal_ceval',
                 name=_name,
-                abbr="ceval-" + _name if _split == "val" else "ceval-test-" + _name,
+                abbr='ceval-' + _name if _split == 'val' else 'ceval-test-' + _name,
                 reader_cfg=ceval_reader_cfg,
                 infer_cfg=ceval_infer_cfg,
                 eval_cfg=ceval_eval_cfg,

configs/datasets/ceval/ceval_ppl_1cd8bf.py

@@ -61,28 +61,28 @@ ceval_subject_mapping = {
 ceval_all_sets = list(ceval_subject_mapping.keys())
 
 ceval_datasets = []
-for _split in ["val", "test"]:
+for _split in ['val', 'test']:
     for _name in ceval_all_sets:
         ceval_reader_cfg = dict(
-            input_columns=["question", "A", "B", "C", "D"],
-            output_column="answer",
-            train_split="dev",
+            input_columns=['question', 'A', 'B', 'C', 'D'],
+            output_column='answer',
+            train_split='dev',
             test_split=_split,
         )
 
         _ch_name = ceval_subject_mapping[_name][1]
 
-        hint = f"以下是关于{_ch_name}的单项选择题，请直接给出正确答案的选项。"
-        question_and_options = "{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}"
+        hint = f'以下是关于{_ch_name}的单项选择题，请直接给出正确答案的选项。'
+        question_and_options = '{question}\nA. {A}\nB. {B}\nC. {C}\nD. {D}'
         ceval_infer_cfg = dict(
             ice_template=dict(
                 type=PromptTemplate,
-                template={answer: f"{question_and_options}\n答案: {answer}\n" for answer in ["A", "B", "C", "D"]},
+                template={answer: f'{question_and_options}\n答案: {answer}\n' for answer in ['A', 'B', 'C', 'D']},
             ),
             prompt_template=dict(
                 type=PromptTemplate,
-                template={answer: f"{hint}\n</E>{question_and_options}\n答案: {answer}" for answer in ["A", "B", "C", "D"]},
-                ice_token="</E>",
+                template={answer: f'{hint}\n</E>{question_and_options}\n答案: {answer}' for answer in ['A', 'B', 'C', 'D']},
+                ice_token='</E>',
             ),
             retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
             inferencer=dict(type=PPLInferencer),
@@ -93,9 +93,9 @@ for _split in ["val", "test"]:
         ceval_datasets.append(
             dict(
                 type=CEvalDataset,
-                path="./data/ceval/formal_ceval",
+                path='./data/ceval/formal_ceval',
                 name=_name,
-                abbr="ceval-" + _name if _split == "val" else "ceval-test-" + _name,
+                abbr='ceval-' + _name if _split == 'val' else 'ceval-test-' + _name,
                 reader_cfg=ceval_reader_cfg,
                 infer_cfg=ceval_infer_cfg,
                 eval_cfg=ceval_eval_cfg,

configs/datasets/ceval/ceval_ppl_578f8d.py

@@ -61,7 +61,7 @@ ceval_subject_mapping = {
 ceval_all_sets = list(ceval_subject_mapping.keys())
 
 ceval_datasets = []
-for _split in ["val"]:
+for _split in ['val']:
     for _name in ceval_all_sets:
         _ch_name = ceval_subject_mapping[_name][1]
         ceval_infer_cfg = dict(
@@ -69,18 +69,18 @@ for _split in ["val"]:
                 type=PromptTemplate,
                 template={
                     answer: dict(
-                        begin="</E>",
+                        begin='</E>',
                         round=[
                             dict(
-                                role="HUMAN",
+                                role='HUMAN',
                                 prompt=
-                                f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: "
+                                f'以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: '
                             ),
-                            dict(role="BOT", prompt=answer),
+                            dict(role='BOT', prompt=answer),
                         ])
-                    for answer in ["A", "B", "C", "D"]
+                    for answer in ['A', 'B', 'C', 'D']
                 },
-                ice_token="</E>",
+                ice_token='</E>',
             ),
             retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
             inferencer=dict(type=PPLInferencer),
@@ -91,14 +91,14 @@ for _split in ["val"]:
         ceval_datasets.append(
             dict(
                 type=CEvalDataset,
-                path="./data/ceval/formal_ceval",
+                path='./data/ceval/formal_ceval',
                 name=_name,
-                abbr="ceval-" + _name if _split == "val" else "ceval-test-" +
+                abbr='ceval-' + _name if _split == 'val' else 'ceval-test-' +
                 _name,
                 reader_cfg=dict(
-                    input_columns=["question", "A", "B", "C", "D"],
-                    output_column="answer",
-                    train_split="dev",
+                    input_columns=['question', 'A', 'B', 'C', 'D'],
+                    output_column='answer',
+                    train_split='dev',
                     test_split=_split),
                 infer_cfg=ceval_infer_cfg,
                 eval_cfg=ceval_eval_cfg,

configs/datasets/ceval/ceval_ppl_93e5ce.py

@@ -61,7 +61,7 @@ ceval_subject_mapping = {
 ceval_all_sets = list(ceval_subject_mapping.keys())
 
 ceval_datasets = []
-for _split in ["val", "test"]:
+for _split in ['val', 'test']:
     for _name in ceval_all_sets:
         _ch_name = ceval_subject_mapping[_name][1]
         ceval_infer_cfg = dict(
@@ -69,18 +69,18 @@ for _split in ["val", "test"]:
                 type=PromptTemplate,
                 template={
                     answer: dict(
-                        begin="</E>",
+                        begin='</E>',
                         round=[
                             dict(
-                                role="HUMAN",
+                                role='HUMAN',
                                 prompt=
-                                f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: "
+                                f'以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: '
                             ),
-                            dict(role="BOT", prompt=answer),
+                            dict(role='BOT', prompt=answer),
                         ])
-                    for answer in ["A", "B", "C", "D"]
+                    for answer in ['A', 'B', 'C', 'D']
                 },
-                ice_token="</E>",
+                ice_token='</E>',
             ),
             retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
             inferencer=dict(type=PPLInferencer),
@@ -91,14 +91,14 @@ for _split in ["val", "test"]:
         ceval_datasets.append(
             dict(
                 type=CEvalDataset,
-                path="./data/ceval/formal_ceval",
+                path='./data/ceval/formal_ceval',
                 name=_name,
-                abbr="ceval-" + _name if _split == "val" else "ceval-test-" +
+                abbr='ceval-' + _name if _split == 'val' else 'ceval-test-' +
                 _name,
                 reader_cfg=dict(
-                    input_columns=["question", "A", "B", "C", "D"],
-                    output_column="answer",
-                    train_split="dev",
+                    input_columns=['question', 'A', 'B', 'C', 'D'],
+                    output_column='answer',
+                    train_split='dev',
                     test_split=_split),
                 infer_cfg=ceval_infer_cfg,
                 eval_cfg=ceval_eval_cfg,

configs/datasets/ceval/ceval_zero_shot_gen_bd40ef.py

@@ -62,23 +62,23 @@ ceval_subject_mapping = {
 ceval_all_sets = list(ceval_subject_mapping.keys())
 
 ceval_datasets = []
-for _split in ["val"]:
+for _split in ['val']:
     for _name in ceval_all_sets:
         _ch_name = ceval_subject_mapping[_name][1]
         ceval_infer_cfg = dict(
             ice_template=dict(
                 type=PromptTemplate,
                 template=dict(
-                    begin="</E>",
+                    begin='</E>',
                     round=[
                         dict(
-                            role="HUMAN",
+                            role='HUMAN',
                             prompt=
-                            f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n让我们一步一步思考。答案: "
+                            f'以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n让我们一步一步思考。答案: '
                         ),
-                        dict(role="BOT", prompt="{answer}"),
+                        dict(role='BOT', prompt='{answer}'),
                     ]),
-                ice_token="</E>",
+                ice_token='</E>',
             ),
             retriever=dict(type=ZeroRetriever),
             inferencer=dict(type=GenInferencer, max_out_len=256),
@@ -91,14 +91,14 @@ for _split in ["val"]:
         ceval_datasets.append(
             dict(
                 type=CEvalDataset,
-                path="./data/ceval/formal_ceval",
+                path='./data/ceval/formal_ceval',
                 name=_name,
-                abbr="ceval-" + _name if _split == "val" else "ceval-test-" +
+                abbr='ceval-' + _name if _split == 'val' else 'ceval-test-' +
                 _name,
                 reader_cfg=dict(
-                    input_columns=["question", "A", "B", "C", "D"],
-                    output_column="answer",
-                    train_split="dev",
+                    input_columns=['question', 'A', 'B', 'C', 'D'],
+                    output_column='answer',
+                    train_split='dev',
                     test_split=_split),
                 infer_cfg=ceval_infer_cfg,
                 eval_cfg=ceval_eval_cfg,

configs/datasets/civilcomments/civilcomments_clp_a3c5fd.py

@@ -15,10 +15,10 @@ civilcomments_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(round=[
             dict(
-                role="HUMAN",
-                prompt="Text: {text}\nQuestion: Does the above text contain "
-                "rude, hateful, aggressive, disrespectful or unreasonable "
-                "language?\nAnswer:")
+                role='HUMAN',
+                prompt='Text: {text}\nQuestion: Does the above text contain '
+                'rude, hateful, aggressive, disrespectful or unreasonable '
+                'language?\nAnswer:')
         ])),
     retriever=dict(type=ZeroRetriever),
     inferencer=dict(type=CLPInferencer))

configs/datasets/clozeTest_maxmin/clozeTest_maxmin_gen_c205fb.py

@@ -7,8 +7,8 @@ from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 
 maxmin_reader_cfg = dict(
-    input_columns=["nl_tokens", "pl_tokens"],
-    output_column="answer",
+    input_columns=['nl_tokens', 'pl_tokens'],
+    output_column='answer',
 )
 
 maxmin_infer_cfg = dict(
@@ -16,8 +16,8 @@ maxmin_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role="HUMAN", prompt="Code:{pl_tokens}\nThe aim of the code: {nl_tokens}\nQuestion: Please tell me what \"<mask>\" in the code should be replaced with and you must response to me only A or B.\nA. max\nB. min\nAnswer:"),
-                dict(role="BOT", prompt="{answer}"),
+                dict(role='HUMAN', prompt="Code:{pl_tokens}\nThe aim of the code: {nl_tokens}\nQuestion: Please tell me what \"<mask>\" in the code should be replaced with and you must response to me only A or B.\nA. max\nB. min\nAnswer:"),
+                dict(role='BOT', prompt='{answer}'),
             ]
         ),
     ),
@@ -26,15 +26,15 @@ maxmin_infer_cfg = dict(
 )
 
 maxmin_eval_cfg = dict(evaluator=dict(type=AccEvaluator),
-                        pred_role="BOT",
+                        pred_role='BOT',
                         pred_postprocessor=dict(type=first_capital_postprocess))
 
 maxmin_datasets = [
     dict(
         type=MaxminDataset,
-        abbr=f"maxmin",
-        test_path=f"data/clozeTest-maxmin/python/clozeTest.json",
-        answer_path=f"data/clozeTest-maxmin/python/answers.txt",
+        abbr=f'maxmin',
+        test_path=f'data/clozeTest-maxmin/python/clozeTest.json',
+        answer_path=f'data/clozeTest-maxmin/python/answers.txt',
         reader_cfg=maxmin_reader_cfg,
         infer_cfg=maxmin_infer_cfg,
         eval_cfg=maxmin_eval_cfg,