[Format] Add config lints (#892)

.pre-commit-config-zh-cn.yaml

@@ -45,7 +45,7 @@ repos:
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/
+              configs/.*?/.*\.txt
             )
       - id: check-yaml
       - id: end-of-file-fixer
@@ -53,11 +53,10 @@ repos:
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/
+              configs/.*?/.*\.txt
             )
       - id: requirements-txt-fixer
       - id: double-quote-string-fixer
-        exclude: configs/
       - id: check-merge-conflict
       - id: fix-encoding-pragma
         args: ["--remove"]

.pre-commit-config.yaml

@@ -45,7 +45,7 @@ repos:
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/
+              configs/.*?/.*\.txt
             )
       - id: check-yaml
       - id: end-of-file-fixer
@@ -53,11 +53,10 @@ repos:
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/
+              configs/.*?/.*\.txt
             )
       - id: requirements-txt-fixer
       - id: double-quote-string-fixer
-        exclude: configs/
       - id: check-merge-conflict
       - id: fix-encoding-pragma
         args: ["--remove"]

configs/api_examples/eval_api_360.py

@@ -17,7 +17,7 @@ models = [
         abbr='360GPT_S2_V9',
         type=AI360GPT,
         path='360GPT_S2_V9',
-        key="xxxxxxxxxxxx",
+        key='xxxxxxxxxxxx',
         generation_kwargs={
             'temperature': 0.9,
             'max_tokens': 2048,
@@ -40,4 +40,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir ="./output/api_360GPT_S2_V9"
\ No newline at end of file
+work_dir ='./output/api_360GPT_S2_V9'

configs/api_examples/eval_api_baichuan.py

@@ -18,8 +18,8 @@ models = [
         type=BaiChuan,
         path='Baichuan2-53B',
         api_key='xxxxxx',
-        secret_key="xxxxx",
-        url="xxxxx",
+        secret_key='xxxxx',
+        url='xxxxx',
         generation_kwargs={
             'temperature': 0.3,
             'top_p': 0.85,
@@ -41,4 +41,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_baichuan53b/"
\ No newline at end of file
+work_dir = 'outputs/api_baichuan53b/'

configs/api_examples/eval_api_baidu.py

@@ -39,4 +39,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_erniebot/"
\ No newline at end of file
+work_dir = 'outputs/api_erniebot/'

configs/api_examples/eval_api_bytedance.py

@@ -18,8 +18,8 @@ models = [
         abbr='skylark-pro-public',
         type=ByteDance,
         path='skylark-pro-public',
-        accesskey="xxxxxxx",
-        secretkey="xxxxxxx",
+        accesskey='xxxxxxx',
+        secretkey='xxxxxxx',
         url='xxxxxx',
         generation_kwargs={
             'temperature': 0.7,
@@ -41,4 +41,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_bytedance/"
\ No newline at end of file
+work_dir = 'outputs/api_bytedance/'

configs/api_examples/eval_api_minimax.py

@@ -34,4 +34,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_minimax/"
\ No newline at end of file
+work_dir = 'outputs/api_minimax/'

configs/api_examples/eval_api_moonshot.py

@@ -37,4 +37,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_moonshot/"
\ No newline at end of file
+work_dir = 'outputs/api_moonshot/'

configs/api_examples/eval_api_nanbeige.py

@@ -18,7 +18,7 @@ models = [
         abbr='nanbeige-plus',
         type=Nanbeige,
         path='nanbeige-plus',
-        key="xxxxxx",
+        key='xxxxxx',
         query_per_second=1,
         max_out_len=2048,
         batch_size=8),
@@ -33,4 +33,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir ="./output/nanbeige-plus"
\ No newline at end of file
+work_dir ='./output/nanbeige-plus'

configs/api_examples/eval_api_pangu.py

@@ -17,13 +17,13 @@ dict(
         abbr='pangu',
         type=PanGu,
         path='pangu',
-        access_key="xxxxxx",
-        secret_key="xxxxxx",
-        url = "xxxxxx",
+        access_key='xxxxxx',
+        secret_key='xxxxxx',
+        url = 'xxxxxx',
         # url of token sever, used for generate token, like "https://xxxxxx.myhuaweicloud.com/v3/auth/tokens",
-        token_url = "xxxxxx",
+        token_url = 'xxxxxx',
         # scope-project-name, used for generate token
-        project_name = "xxxxxx",
+        project_name = 'xxxxxx',
         query_per_second=1,
         max_out_len=2048,
         max_seq_len=2048,
@@ -39,4 +39,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_pangu/"
\ No newline at end of file
+work_dir = 'outputs/api_pangu/'

configs/api_examples/eval_api_qwen.py

@@ -37,4 +37,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_qwen/"
+work_dir = 'outputs/api_qwen/'

configs/api_examples/eval_api_sensetime.py

@@ -24,17 +24,17 @@ models = [
         max_seq_len=2048,
         batch_size=8,
         parameters={
-            "temperature": 0.8,
-            "top_p": 0.7,
-            "max_new_tokens": 1024,
-            "repetition_penalty": 1.05,
-            "know_ids": [],
-            "stream": True,
-            "user": "#*#***TestUser***#*#",
-            "knowledge_config": {
-                "control_level": "normal",
-                "knowledge_base_result": False,
-                "online_search_result": False
+            'temperature': 0.8,
+            'top_p': 0.7,
+            'max_new_tokens': 1024,
+            'repetition_penalty': 1.05,
+            'know_ids': [],
+            'stream': True,
+            'user': '#*#***TestUser***#*#',
+            'knowledge_config': {
+                'control_level': 'normal',
+                'knowledge_base_result': False,
+                'online_search_result': False
             }
         }
     )
@@ -49,4 +49,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_sensetime/"
\ No newline at end of file
+work_dir = 'outputs/api_sensetime/'

configs/api_examples/eval_api_xunfei.py

@@ -17,10 +17,10 @@ models = [
     dict(
         abbr='Spark-v1-1',
         type=XunFei,
-        appid="xxxx",
+        appid='xxxx',
         path='ws://spark-api.xf-yun.com/v1.1/chat',
-        api_secret = "xxxxxxx",
-        api_key = "xxxxxxx",
+        api_secret = 'xxxxxxx',
+        api_key = 'xxxxxxx',
         query_per_second=1,
         max_out_len=2048,
         max_seq_len=2048,
@@ -28,11 +28,11 @@ models = [
     dict(
         abbr='Spark-v3-1',
         type=XunFei,
-        appid="xxxx",
+        appid='xxxx',
         domain='generalv3',
         path='ws://spark-api.xf-yun.com/v3.1/chat',
-        api_secret = "xxxxxxxx",
-        api_key = "xxxxxxxxx",
+        api_secret = 'xxxxxxxx',
+        api_key = 'xxxxxxxxx',
         query_per_second=1,
         max_out_len=2048,
         max_seq_len=2048,
@@ -48,4 +48,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_xunfei/"
\ No newline at end of file
+work_dir = 'outputs/api_xunfei/'

configs/api_examples/eval_api_zhipu.py

@@ -29,7 +29,7 @@ models = [
         abbr='chatglm_pro',
         type=ZhiPuAI,
         path='chatglm_pro',
-        key='xxxxxxxxxxxx', 
+        key='xxxxxxxxxxxx',
         query_per_second=1,
         max_out_len=2048,
         max_seq_len=2048,
@@ -45,4 +45,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_zhipu/"
\ No newline at end of file
+work_dir = 'outputs/api_zhipu/'

configs/api_examples/eval_api_zhipu_v2.py

@@ -64,4 +64,4 @@ infer = dict(
         task=dict(type=OpenICLInferTask)),
 )
 
-work_dir = "outputs/api_zhipu_v2/"
\ No newline at end of file
+work_dir = 'outputs/api_zhipu_v2/'

configs/dataset_collections/chat_OC15.py

@@ -19,4 +19,4 @@ with read_base():
     from ..datasets.gpqa.gpqa_gen_4baadb import gpqa_datasets
     from ..datasets.IFEval.IFEval_gen_3321a3 import ifeval_datasets
 
-datasets = sum((v for k, v in locals().items() if k.endswith("_datasets")), [])
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

configs/datasets/ARC_c/ARC_c_clean_ppl.py

@@ -12,29 +12,29 @@ ARC_c_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template={
-            "A":
+            'A':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textA}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textA}')
                 ], ),
-            "B":
+            'B':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textB}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textB}')
                 ], ),
-            "C":
+            'C':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textC}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textC}')
                 ], ),
-            "D":
+            'D':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textD}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textD}')
                 ], ),
         }),
     retriever=dict(type=ZeroRetriever),

configs/datasets/ARC_c/ARC_c_gen_1e0de5.py

@@ -6,8 +6,8 @@ from opencompass.datasets import ARCDataset
 from opencompass.utils.text_postprocessors import first_option_postprocess
 
 ARC_c_reader_cfg = dict(
-    input_columns=["question", "textA", "textB", "textC", "textD"],
-    output_column="answerKey")
+    input_columns=['question', 'textA', 'textB', 'textC', 'textD'],
+    output_column='answerKey')
 
 ARC_c_infer_cfg = dict(
     prompt_template=dict(
@@ -15,9 +15,9 @@ ARC_c_infer_cfg = dict(
         template=dict(
             round=[
                 dict(
-                    role="HUMAN",
+                    role='HUMAN',
                     prompt=
-                    "Question: {question}\nA. {textA}\nB. {textB}\nC. {textC}\nD. {textD}\nAnswer:"
+                    'Question: {question}\nA. {textA}\nB. {textB}\nC. {textC}\nD. {textD}\nAnswer:'
                 )
             ], ),
     ),
@@ -27,15 +27,15 @@ ARC_c_infer_cfg = dict(
 
 ARC_c_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),
-    pred_role="BOT",
+    pred_role='BOT',
     pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
 )
 
 ARC_c_datasets = [
     dict(
-        abbr="ARC-c",
+        abbr='ARC-c',
         type=ARCDataset,
-        path="./data/ARC/ARC-c/ARC-Challenge-Dev.jsonl",
+        path='./data/ARC/ARC-c/ARC-Challenge-Dev.jsonl',
         reader_cfg=ARC_c_reader_cfg,
         infer_cfg=ARC_c_infer_cfg,
         eval_cfg=ARC_c_eval_cfg,

configs/datasets/ARC_c/ARC_c_ppl_2ef631.py

@@ -14,10 +14,10 @@ ARC_c_infer_cfg = dict(
         template={
             opt: dict(
                 round=[
-                    dict(role="HUMAN", prompt=f"{{question}}\nA. {{textA}}\nB. {{textB}}\nC. {{textC}}\nD. {{textD}}"),
-                    dict(role="BOT", prompt=f"Answer: {opt}"),
+                    dict(role='HUMAN', prompt=f'{{question}}\nA. {{textA}}\nB. {{textB}}\nC. {{textC}}\nD. {{textD}}'),
+                    dict(role='BOT', prompt=f'Answer: {opt}'),
                 ]
-            ) for opt in ["A", "B", "C", "D"]
+            ) for opt in ['A', 'B', 'C', 'D']
         },
     ),
     retriever=dict(type=ZeroRetriever),

configs/datasets/ARC_c/ARC_c_ppl_a450bd.py

@@ -12,29 +12,29 @@ ARC_c_infer_cfg = dict(
     prompt_template=dict(
         type=PromptTemplate,
         template={
-            "A":
+            'A':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textA}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textA}')
                 ], ),
-            "B":
+            'B':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textB}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textB}')
                 ], ),
-            "C":
+            'C':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textC}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textC}')
                 ], ),
-            "D":
+            'D':
             dict(
                 round=[
-                    dict(role="HUMAN", prompt="Question: {question}\nAnswer: "),
-                    dict(role="BOT", prompt="{textD}")
+                    dict(role='HUMAN', prompt='Question: {question}\nAnswer: '),
+                    dict(role='BOT', prompt='{textD}')
                 ], ),
         }),
     retriever=dict(type=ZeroRetriever),